2024-12-08 21:13:57,278 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 21:13:57,299 main DEBUG Took 0.018522 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-08 21:13:57,299 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-08 21:13:57,300 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-08 21:13:57,301 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-08 21:13:57,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,315 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-08 21:13:57,333 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,335 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,336 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,337 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,338 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,339 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,340 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,341 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,341 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,342 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,342 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,343 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,344 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,344 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,344 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,345 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,345 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,346 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,346 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-08 21:13:57,347 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,347 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-08 21:13:57,349 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-08 21:13:57,352 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-08 21:13:57,355 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-08 21:13:57,355 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-08 21:13:57,357 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-08 21:13:57,357 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-08 21:13:57,368 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-08 21:13:57,371 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-08 21:13:57,373 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-08 21:13:57,373 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-08 21:13:57,374 main DEBUG createAppenders(={Console}) 2024-12-08 21:13:57,375 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-08 21:13:57,375 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-08 21:13:57,376 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-08 21:13:57,376 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-08 21:13:57,377 main DEBUG OutputStream closed 2024-12-08 21:13:57,377 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-08 21:13:57,377 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-08 21:13:57,378 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-08 21:13:57,468 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-08 21:13:57,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-08 21:13:57,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-08 21:13:57,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-08 21:13:57,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-08 21:13:57,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-08 21:13:57,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-08 21:13:57,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-08 21:13:57,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-08 21:13:57,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-08 21:13:57,481 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-08 21:13:57,481 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-08 21:13:57,482 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-08 21:13:57,482 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-08 21:13:57,482 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-08 21:13:57,483 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-08 21:13:57,483 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-08 21:13:57,484 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-08 21:13:57,486 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08 21:13:57,486 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-08 21:13:57,487 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-08 21:13:57,488 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-08T21:13:57,763 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd 2024-12-08 21:13:57,767 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-08 21:13:57,767 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-08T21:13:57,777 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-08T21:13:57,793 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-08T21:13:57,810 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611, deleteOnExit=true 2024-12-08T21:13:57,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-08T21:13:57,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/test.cache.data in system properties and HBase conf 2024-12-08T21:13:57,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/hadoop.tmp.dir in system properties and HBase conf 2024-12-08T21:13:57,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/hadoop.log.dir in system properties and HBase conf 2024-12-08T21:13:57,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-08T21:13:57,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-08T21:13:57,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-08T21:13:57,894 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-08T21:13:58,004 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-08T21:13:58,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-08T21:13:58,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-08T21:13:58,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-08T21:13:58,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T21:13:58,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-08T21:13:58,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-08T21:13:58,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-08T21:13:58,013 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T21:13:58,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-08T21:13:58,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/nfs.dump.dir in system properties and HBase conf 2024-12-08T21:13:58,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/java.io.tmpdir in system properties and HBase conf 2024-12-08T21:13:58,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-08T21:13:58,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-08T21:13:58,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-08T21:13:59,392 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-08T21:13:59,465 INFO [Time-limited test {}] log.Log(170): Logging initialized @2956ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-08T21:13:59,526 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T21:13:59,577 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T21:13:59,596 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T21:13:59,596 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T21:13:59,597 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-08T21:13:59,610 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T21:13:59,612 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/hadoop.log.dir/,AVAILABLE} 2024-12-08T21:13:59,614 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T21:13:59,775 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/java.io.tmpdir/jetty-localhost-43491-hadoop-hdfs-3_4_1-tests_jar-_-any-2371873318957891916/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T21:13:59,786 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43491} 2024-12-08T21:13:59,786 INFO [Time-limited test {}] server.Server(415): Started @3277ms 2024-12-08T21:14:00,573 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-08T21:14:00,581 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-08T21:14:00,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-08T21:14:00,582 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-08T21:14:00,582 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-08T21:14:00,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/hadoop.log.dir/,AVAILABLE} 2024-12-08T21:14:00,584 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-08T21:14:00,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/java.io.tmpdir/jetty-localhost-44739-hadoop-hdfs-3_4_1-tests_jar-_-any-7820655173584130177/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T21:14:00,692 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:44739} 2024-12-08T21:14:00,692 INFO [Time-limited test {}] server.Server(415): Started @4183ms 2024-12-08T21:14:00,741 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-08T21:14:02,302 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611/data/data2/current/BP-1093270908-172.17.0.2-1733692438514/current, will proceed with Du for space computation calculation, 2024-12-08T21:14:02,302 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611/data/data1/current/BP-1093270908-172.17.0.2-1733692438514/current, will proceed with Du for space computation calculation, 2024-12-08T21:14:02,326 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-08T21:14:02,367 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e41b7330958aa7a with lease ID 0x732aa184fb0f55a3: Processing first storage report for DS-53f29116-7117-4010-9449-64716e9d1b3b from datanode DatanodeRegistration(127.0.0.1:42767, datanodeUuid=201a3fb0-05db-4bd4-b37f-702c1891983f, infoPort=39271, infoSecurePort=0, ipcPort=38903, storageInfo=lv=-57;cid=testClusterID;nsid=1283664913;c=1733692438514) 2024-12-08T21:14:02,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e41b7330958aa7a with lease ID 0x732aa184fb0f55a3: from storage DS-53f29116-7117-4010-9449-64716e9d1b3b node DatanodeRegistration(127.0.0.1:42767, datanodeUuid=201a3fb0-05db-4bd4-b37f-702c1891983f, infoPort=39271, infoSecurePort=0, ipcPort=38903, storageInfo=lv=-57;cid=testClusterID;nsid=1283664913;c=1733692438514), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-08T21:14:02,369 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e41b7330958aa7a with lease ID 0x732aa184fb0f55a3: Processing first storage report for DS-12e4ce57-10b5-4b5a-a7bc-2865bffbf67c from datanode DatanodeRegistration(127.0.0.1:42767, datanodeUuid=201a3fb0-05db-4bd4-b37f-702c1891983f, infoPort=39271, infoSecurePort=0, ipcPort=38903, storageInfo=lv=-57;cid=testClusterID;nsid=1283664913;c=1733692438514) 2024-12-08T21:14:02,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e41b7330958aa7a with lease ID 0x732aa184fb0f55a3: from storage DS-12e4ce57-10b5-4b5a-a7bc-2865bffbf67c node DatanodeRegistration(127.0.0.1:42767, datanodeUuid=201a3fb0-05db-4bd4-b37f-702c1891983f, infoPort=39271, infoSecurePort=0, ipcPort=38903, storageInfo=lv=-57;cid=testClusterID;nsid=1283664913;c=1733692438514), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-08T21:14:02,446 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd 2024-12-08T21:14:02,548 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611/zookeeper_0, clientPort=56209, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-08T21:14:02,557 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56209 2024-12-08T21:14:02,566 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:14:02,569 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:14:02,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741825_1001 (size=7) 2024-12-08T21:14:03,243 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd with version=8 2024-12-08T21:14:03,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/hbase-staging 2024-12-08T21:14:03,317 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-08T21:14:03,532 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e653e208f502:0 server-side Connection retries=45 2024-12-08T21:14:03,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:14:03,541 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:14:03,544 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:14:03,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:14:03,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:14:03,650 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-08T21:14:03,698 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-08T21:14:03,705 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-08T21:14:03,709 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:14:03,729 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4666 (auto-detected) 2024-12-08T21:14:03,729 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-08T21:14:03,745 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45981 2024-12-08T21:14:03,763 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45981 connecting to ZooKeeper ensemble=127.0.0.1:56209 2024-12-08T21:14:03,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459810x0, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:14:03,878 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45981-0x10007474e6b0000 connected 2024-12-08T21:14:03,964 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:14:03,968 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:14:03,978 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:14:03,981 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd, hbase.cluster.distributed=false 2024-12-08T21:14:04,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:14:04,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45981 2024-12-08T21:14:04,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45981 2024-12-08T21:14:04,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45981 2024-12-08T21:14:04,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45981 2024-12-08T21:14:04,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45981 2024-12-08T21:14:04,107 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e653e208f502:0 server-side Connection retries=45 2024-12-08T21:14:04,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:14:04,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-08T21:14:04,109 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-08T21:14:04,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-08T21:14:04,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-08T21:14:04,111 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-08T21:14:04,113 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-08T21:14:04,114 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38457 2024-12-08T21:14:04,116 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38457 connecting to ZooKeeper ensemble=127.0.0.1:56209 2024-12-08T21:14:04,117 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:14:04,120 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:14:04,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384570x0, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-08T21:14:04,179 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:384570x0, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:14:04,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38457-0x10007474e6b0001 connected 2024-12-08T21:14:04,184 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-08T21:14:04,193 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-08T21:14:04,195 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-08T21:14:04,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-08T21:14:04,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38457 2024-12-08T21:14:04,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38457 2024-12-08T21:14:04,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38457 2024-12-08T21:14:04,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38457 2024-12-08T21:14:04,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38457 2024-12-08T21:14:04,214 DEBUG [M:0;e653e208f502:45981 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e653e208f502:45981 2024-12-08T21:14:04,215 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e653e208f502,45981,1733692443391 2024-12-08T21:14:04,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:14:04,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:14:04,248 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e653e208f502,45981,1733692443391 2024-12-08T21:14:04,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-08T21:14:04,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:04,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:04,277 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-08T21:14:04,278 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e653e208f502,45981,1733692443391 from backup master directory 2024-12-08T21:14:04,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:14:04,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e653e208f502,45981,1733692443391 2024-12-08T21:14:04,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-08T21:14:04,287 WARN [master/e653e208f502:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:14:04,288 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e653e208f502,45981,1733692443391 2024-12-08T21:14:04,289 INFO [master/e653e208f502:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-08T21:14:04,290 INFO [master/e653e208f502:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-08T21:14:04,337 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/hbase.id] with ID: 4a8351f1-229c-4b50-b24b-8c67e222f303 2024-12-08T21:14:04,337 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/.tmp/hbase.id 2024-12-08T21:14:04,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741826_1002 (size=42) 2024-12-08T21:14:04,749 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/.tmp/hbase.id]:[hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/hbase.id] 2024-12-08T21:14:04,796 INFO [master/e653e208f502:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-08T21:14:04,802 INFO [master/e653e208f502:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-08T21:14:04,821 INFO [master/e653e208f502:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-08T21:14:04,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:04,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:04,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741827_1003 (size=196) 2024-12-08T21:14:05,309 INFO [master/e653e208f502:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:14:05,311 INFO [master/e653e208f502:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-08T21:14:05,324 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:05,328 INFO [master/e653e208f502:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:14:05,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741828_1004 (size=1189) 2024-12-08T21:14:05,776 INFO [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store 2024-12-08T21:14:05,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741829_1005 (size=34) 2024-12-08T21:14:06,209 INFO [master/e653e208f502:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-08T21:14:06,212 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:06,213 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:14:06,213 INFO [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:14:06,214 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:14:06,215 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:14:06,215 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:14:06,216 INFO [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:14:06,217 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733692446213Disabling compacts and flushes for region at 1733692446213Disabling writes for close at 1733692446215 (+2 ms)Writing region close event to WAL at 1733692446216 (+1 ms)Closed at 1733692446216 2024-12-08T21:14:06,220 WARN [master/e653e208f502:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/.initializing 2024-12-08T21:14:06,220 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/WALs/e653e208f502,45981,1733692443391 2024-12-08T21:14:06,227 INFO [master/e653e208f502:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:14:06,240 INFO [master/e653e208f502:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e653e208f502%2C45981%2C1733692443391, suffix=, logDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/WALs/e653e208f502,45981,1733692443391, archiveDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/oldWALs, maxLogs=10 2024-12-08T21:14:06,259 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/WALs/e653e208f502,45981,1733692443391/e653e208f502%2C45981%2C1733692443391.1733692446244, exclude list is [], retry=0 2024-12-08T21:14:06,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42767,DS-53f29116-7117-4010-9449-64716e9d1b3b,DISK] 2024-12-08T21:14:06,276 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-08T21:14:06,308 INFO [master/e653e208f502:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/WALs/e653e208f502,45981,1733692443391/e653e208f502%2C45981%2C1733692443391.1733692446244 2024-12-08T21:14:06,309 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39271:39271)] 2024-12-08T21:14:06,309 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:14:06,310 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:06,312 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,313 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-08T21:14:06,368 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:06,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:06,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-08T21:14:06,374 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:06,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:06,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,378 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-08T21:14:06,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:06,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:06,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-08T21:14:06,382 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:06,383 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:06,383 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,387 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,388 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,392 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,393 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,397 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-08T21:14:06,400 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-08T21:14:06,405 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:14:06,406 INFO [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72067207, jitterRate=0.07388506829738617}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-08T21:14:06,413 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733692446324Initializing all the Stores at 1733692446326 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733692446327 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692446327Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692446327Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692446327Cleaning up temporary data from old regions at 1733692446393 (+66 ms)Region opened successfully at 1733692446413 (+20 ms) 2024-12-08T21:14:06,415 INFO [master/e653e208f502:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-08T21:14:06,451 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18e0b02e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e653e208f502/172.17.0.2:0 2024-12-08T21:14:06,481 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-08T21:14:06,490 INFO [master/e653e208f502:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-08T21:14:06,490 INFO [master/e653e208f502:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-08T21:14:06,493 INFO [master/e653e208f502:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-08T21:14:06,495 INFO [master/e653e208f502:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-08T21:14:06,499 INFO [master/e653e208f502:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-08T21:14:06,499 INFO [master/e653e208f502:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-08T21:14:06,524 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-08T21:14:06,534 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-08T21:14:06,643 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-08T21:14:06,647 INFO [master/e653e208f502:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-08T21:14:06,649 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-08T21:14:06,655 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-08T21:14:06,657 INFO [master/e653e208f502:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-08T21:14:06,662 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-08T21:14:06,674 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-08T21:14:06,677 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-08T21:14:06,687 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-08T21:14:06,710 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-08T21:14:06,718 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-08T21:14:06,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:14:06,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-08T21:14:06,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:06,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:06,732 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e653e208f502,45981,1733692443391, sessionid=0x10007474e6b0000, setting cluster-up flag (Was=false) 2024-12-08T21:14:06,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:06,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:06,792 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-08T21:14:06,797 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e653e208f502,45981,1733692443391 2024-12-08T21:14:06,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:06,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:06,855 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-08T21:14:06,860 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e653e208f502,45981,1733692443391 2024-12-08T21:14:06,868 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-08T21:14:06,908 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(746): ClusterId : 4a8351f1-229c-4b50-b24b-8c67e222f303 2024-12-08T21:14:06,910 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-08T21:14:06,920 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-08T21:14:06,920 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-08T21:14:06,930 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-08T21:14:06,930 DEBUG [RS:0;e653e208f502:38457 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4755102b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e653e208f502/172.17.0.2:0 2024-12-08T21:14:06,932 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-08T21:14:06,941 INFO [master/e653e208f502:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-08T21:14:06,942 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e653e208f502:38457 2024-12-08T21:14:06,945 INFO [RS:0;e653e208f502:38457 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-08T21:14:06,945 INFO [RS:0;e653e208f502:38457 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-08T21:14:06,945 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-08T21:14:06,947 INFO [master/e653e208f502:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-08T21:14:06,948 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(2659): reportForDuty to master=e653e208f502,45981,1733692443391 with port=38457, startcode=1733692444075 2024-12-08T21:14:06,951 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e653e208f502,45981,1733692443391 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-08T21:14:06,958 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e653e208f502:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:14:06,958 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e653e208f502:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:14:06,958 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e653e208f502:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:14:06,958 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e653e208f502:0, corePoolSize=5, maxPoolSize=5 2024-12-08T21:14:06,958 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e653e208f502:0, corePoolSize=10, maxPoolSize=10 2024-12-08T21:14:06,958 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:06,959 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e653e208f502:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:14:06,959 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:06,959 DEBUG [RS:0;e653e208f502:38457 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-08T21:14:06,961 INFO [master/e653e208f502:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733692476960 2024-12-08T21:14:06,962 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-08T21:14:06,963 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-08T21:14:06,964 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:14:06,964 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-08T21:14:06,966 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-08T21:14:06,967 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-08T21:14:06,967 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-08T21:14:06,967 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-08T21:14:06,968 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:06,970 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:06,970 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-08T21:14:06,971 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-08T21:14:06,972 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-08T21:14:06,972 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-08T21:14:06,975 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-08T21:14:06,976 INFO [master/e653e208f502:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-08T21:14:06,978 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e653e208f502:0:becomeActiveMaster-HFileCleaner.large.0-1733692446977,5,FailOnTimeoutGroup] 2024-12-08T21:14:06,979 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e653e208f502:0:becomeActiveMaster-HFileCleaner.small.0-1733692446978,5,FailOnTimeoutGroup] 2024-12-08T21:14:06,979 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:06,979 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-08T21:14:06,981 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:06,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741831_1007 (size=1321) 2024-12-08T21:14:06,982 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:06,991 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41625, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-08T21:14:06,995 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45981 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e653e208f502,38457,1733692444075 2024-12-08T21:14:06,997 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45981 {}] master.ServerManager(517): Registering regionserver=e653e208f502,38457,1733692444075 2024-12-08T21:14:07,009 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:14:07,009 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36411 2024-12-08T21:14:07,009 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-08T21:14:07,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:14:07,109 DEBUG [RS:0;e653e208f502:38457 {}] zookeeper.ZKUtil(111): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e653e208f502,38457,1733692444075 2024-12-08T21:14:07,109 WARN [RS:0;e653e208f502:38457 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-08T21:14:07,110 INFO [RS:0;e653e208f502:38457 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:14:07,111 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/WALs/e653e208f502,38457,1733692444075 2024-12-08T21:14:07,114 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e653e208f502,38457,1733692444075] 2024-12-08T21:14:07,136 INFO [RS:0;e653e208f502:38457 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-08T21:14:07,147 INFO [RS:0;e653e208f502:38457 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-08T21:14:07,151 INFO [RS:0;e653e208f502:38457 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-08T21:14:07,152 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,153 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-08T21:14:07,157 INFO [RS:0;e653e208f502:38457 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-08T21:14:07,158 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e653e208f502:0, corePoolSize=2, maxPoolSize=2 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,159 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,160 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,160 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,160 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,160 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e653e208f502:0, corePoolSize=1, maxPoolSize=1 2024-12-08T21:14:07,160 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e653e208f502:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:14:07,160 DEBUG [RS:0;e653e208f502:38457 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0, corePoolSize=3, maxPoolSize=3 2024-12-08T21:14:07,161 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,161 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,161 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,162 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,162 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,162 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,38457,1733692444075-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:14:07,177 INFO [RS:0;e653e208f502:38457 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-08T21:14:07,179 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,38457,1733692444075-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,179 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,179 INFO [RS:0;e653e208f502:38457 {}] regionserver.Replication(171): e653e208f502,38457,1733692444075 started 2024-12-08T21:14:07,194 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:07,194 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1482): Serving as e653e208f502,38457,1733692444075, RpcServer on e653e208f502/172.17.0.2:38457, sessionid=0x10007474e6b0001 2024-12-08T21:14:07,195 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-08T21:14:07,195 DEBUG [RS:0;e653e208f502:38457 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e653e208f502,38457,1733692444075 2024-12-08T21:14:07,195 DEBUG [RS:0;e653e208f502:38457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e653e208f502,38457,1733692444075' 2024-12-08T21:14:07,195 DEBUG [RS:0;e653e208f502:38457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-08T21:14:07,196 DEBUG [RS:0;e653e208f502:38457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-08T21:14:07,197 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-08T21:14:07,197 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-08T21:14:07,197 DEBUG [RS:0;e653e208f502:38457 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e653e208f502,38457,1733692444075 2024-12-08T21:14:07,197 DEBUG [RS:0;e653e208f502:38457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e653e208f502,38457,1733692444075' 2024-12-08T21:14:07,197 DEBUG [RS:0;e653e208f502:38457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-08T21:14:07,198 DEBUG [RS:0;e653e208f502:38457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-08T21:14:07,199 DEBUG [RS:0;e653e208f502:38457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-08T21:14:07,199 INFO [RS:0;e653e208f502:38457 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-08T21:14:07,199 INFO [RS:0;e653e208f502:38457 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-08T21:14:07,308 INFO [RS:0;e653e208f502:38457 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-08T21:14:07,312 INFO [RS:0;e653e208f502:38457 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e653e208f502%2C38457%2C1733692444075, suffix=, logDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/WALs/e653e208f502,38457,1733692444075, archiveDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/oldWALs, maxLogs=32 2024-12-08T21:14:07,326 DEBUG [RS:0;e653e208f502:38457 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/WALs/e653e208f502,38457,1733692444075/e653e208f502%2C38457%2C1733692444075.1733692447315, exclude list is [], retry=0 2024-12-08T21:14:07,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42767,DS-53f29116-7117-4010-9449-64716e9d1b3b,DISK] 2024-12-08T21:14:07,333 INFO [RS:0;e653e208f502:38457 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/WALs/e653e208f502,38457,1733692444075/e653e208f502%2C38457%2C1733692444075.1733692447315 2024-12-08T21:14:07,334 DEBUG [RS:0;e653e208f502:38457 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39271:39271)] 2024-12-08T21:14:07,385 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-08T21:14:07,386 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:14:07,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741833_1009 (size=32) 2024-12-08T21:14:07,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:07,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T21:14:07,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:14:07,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:07,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:07,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T21:14:07,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:14:07,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:07,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:07,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T21:14:07,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:14:07,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:07,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:07,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T21:14:07,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:14:07,819 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:07,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:07,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:14:07,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740 2024-12-08T21:14:07,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740 2024-12-08T21:14:07,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:14:07,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:14:07,829 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:14:07,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:14:07,835 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:14:07,836 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71977186, jitterRate=0.07254365086555481}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:14:07,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733692447803Initializing all the Stores at 1733692447806 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733692447806Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733692447807 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692447807Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733692447807Cleaning up temporary data from old regions at 1733692447828 (+21 ms)Region opened successfully at 1733692447838 (+10 ms) 2024-12-08T21:14:07,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:14:07,839 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:14:07,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:14:07,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:14:07,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:14:07,840 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:14:07,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733692447838Disabling compacts and flushes for region at 1733692447838Disabling writes for close at 1733692447839 (+1 ms)Writing region close event to WAL at 1733692447840 (+1 ms)Closed at 1733692447840 2024-12-08T21:14:07,844 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:14:07,844 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-08T21:14:07,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-08T21:14:07,856 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:14:07,859 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-08T21:14:08,012 DEBUG [e653e208f502:45981 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-08T21:14:08,026 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:08,032 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e653e208f502,38457,1733692444075, state=OPENING 2024-12-08T21:14:08,087 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-08T21:14:08,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:08,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:14:08,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:14:08,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:14:08,100 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-08T21:14:08,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:14:08,280 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-08T21:14:08,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60789, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-08T21:14:08,292 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-08T21:14:08,292 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-08T21:14:08,293 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-08T21:14:08,296 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e653e208f502%2C38457%2C1733692444075.meta, suffix=.meta, logDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/WALs/e653e208f502,38457,1733692444075, archiveDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/oldWALs, maxLogs=32 2024-12-08T21:14:08,309 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/WALs/e653e208f502,38457,1733692444075/e653e208f502%2C38457%2C1733692444075.meta.1733692448297.meta, exclude list is [], retry=0 2024-12-08T21:14:08,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42767,DS-53f29116-7117-4010-9449-64716e9d1b3b,DISK] 2024-12-08T21:14:08,315 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/WALs/e653e208f502,38457,1733692444075/e653e208f502%2C38457%2C1733692444075.meta.1733692448297.meta 2024-12-08T21:14:08,315 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39271:39271)] 2024-12-08T21:14:08,316 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:14:08,317 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-08T21:14:08,319 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-08T21:14:08,324 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-08T21:14:08,329 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-08T21:14:08,330 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:08,330 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-08T21:14:08,330 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-08T21:14:08,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-08T21:14:08,335 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-08T21:14:08,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:08,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:08,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-08T21:14:08,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-08T21:14:08,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:08,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:08,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-08T21:14:08,340 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-08T21:14:08,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:08,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:08,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-08T21:14:08,342 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-08T21:14:08,342 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:08,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-08T21:14:08,343 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-08T21:14:08,345 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740 2024-12-08T21:14:08,348 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740 2024-12-08T21:14:08,350 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-08T21:14:08,351 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-08T21:14:08,351 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:14:08,354 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-08T21:14:08,356 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67680476, jitterRate=0.008517682552337646}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:14:08,356 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-08T21:14:08,358 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733692448331Writing region info on filesystem at 1733692448331Initializing all the Stores at 1733692448333 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733692448333Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733692448333Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692448333Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733692448333Cleaning up temporary data from old regions at 1733692448351 (+18 ms)Running coprocessor post-open hooks at 1733692448356 (+5 ms)Region opened successfully at 1733692448358 (+2 ms) 2024-12-08T21:14:08,365 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733692448270 2024-12-08T21:14:08,376 DEBUG [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-08T21:14:08,376 INFO [RS_OPEN_META-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-08T21:14:08,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:08,381 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e653e208f502,38457,1733692444075, state=OPEN 2024-12-08T21:14:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:14:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-08T21:14:08,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:14:08,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-08T21:14:08,418 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:08,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-08T21:14:08,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e653e208f502,38457,1733692444075 in 316 msec 2024-12-08T21:14:08,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-08T21:14:08,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 576 msec 2024-12-08T21:14:08,433 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-08T21:14:08,434 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-08T21:14:08,450 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:08,452 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:08,470 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:08,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60845, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:08,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5960 sec 2024-12-08T21:14:08,492 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733692448492, completionTime=-1 2024-12-08T21:14:08,495 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-08T21:14:08,496 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-08T21:14:08,523 INFO [master/e653e208f502:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-08T21:14:08,523 INFO [master/e653e208f502:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733692508523 2024-12-08T21:14:08,523 INFO [master/e653e208f502:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733692568523 2024-12-08T21:14:08,523 INFO [master/e653e208f502:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-08T21:14:08,525 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,45981,1733692443391-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:08,526 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,45981,1733692443391-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:08,526 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,45981,1733692443391-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:08,527 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e653e208f502:45981, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:08,528 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:08,528 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:08,534 DEBUG [master/e653e208f502:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-08T21:14:08,554 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.266sec 2024-12-08T21:14:08,555 INFO [master/e653e208f502:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-08T21:14:08,556 INFO [master/e653e208f502:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-08T21:14:08,557 INFO [master/e653e208f502:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-08T21:14:08,557 INFO [master/e653e208f502:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-08T21:14:08,557 INFO [master/e653e208f502:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-08T21:14:08,558 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,45981,1733692443391-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-08T21:14:08,559 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,45981,1733692443391-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-08T21:14:08,566 DEBUG [master/e653e208f502:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-08T21:14:08,567 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-08T21:14:08,567 INFO [master/e653e208f502:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e653e208f502,45981,1733692443391-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-08T21:14:08,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:08,620 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-08T21:14:08,620 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-08T21:14:08,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:08,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:08,632 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db5a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:08,642 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:08,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:08,645 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:08,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:08,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:08,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e653e208f502,45981,1733692443391 2024-12-08T21:14:08,685 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=8391 2024-12-08T21:14:08,694 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:14:08,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e653e208f502,45981,1733692443391 2024-12-08T21:14:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f523d87 2024-12-08T21:14:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:14:08,719 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:14:08,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:14:08,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:14:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:08,742 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:14:08,744 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-08T21:14:08,744 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:08,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:14:08,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:14:08,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741835_1011 (size=1061) 2024-12-08T21:14:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:14:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:14:09,165 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:14:09,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741836_1012 (size=53) 2024-12-08T21:14:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:14:09,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:09,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0ef41e10da56e9b6857487fb3e409164, disabling compactions & flushes 2024-12-08T21:14:09,577 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. after waiting 0 ms 2024-12-08T21:14:09,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,577 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0ef41e10da56e9b6857487fb3e409164: Waiting for close lock at 1733692449577Disabling compacts and flushes for region at 1733692449577Disabling writes for close at 1733692449577Writing region close event to WAL at 1733692449577Closed at 1733692449577 2024-12-08T21:14:09,580 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:14:09,587 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733692449580"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733692449580"}]},"ts":"1733692449580"} 2024-12-08T21:14:09,590 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:14:09,592 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:14:09,594 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692449592"}]},"ts":"1733692449592"} 2024-12-08T21:14:09,598 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T21:14:09,599 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, ASSIGN}] 2024-12-08T21:14:09,602 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, ASSIGN 2024-12-08T21:14:09,603 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, ASSIGN; state=OFFLINE, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=false 2024-12-08T21:14:09,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ef41e10da56e9b6857487fb3e409164, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:09,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, ASSIGN because future has completed 2024-12-08T21:14:09,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:14:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:14:09,928 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,929 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:14:09,929 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,930 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:09,930 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,930 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,933 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,937 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:09,937 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ef41e10da56e9b6857487fb3e409164 columnFamilyName A 2024-12-08T21:14:09,938 DEBUG [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:09,939 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.HStore(327): Store=0ef41e10da56e9b6857487fb3e409164/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:09,939 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,941 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:09,941 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ef41e10da56e9b6857487fb3e409164 columnFamilyName B 2024-12-08T21:14:09,941 DEBUG [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:09,942 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.HStore(327): Store=0ef41e10da56e9b6857487fb3e409164/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:09,942 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,944 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:09,944 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0ef41e10da56e9b6857487fb3e409164 columnFamilyName C 2024-12-08T21:14:09,944 DEBUG [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:09,945 INFO [StoreOpener-0ef41e10da56e9b6857487fb3e409164-1 {}] regionserver.HStore(327): Store=0ef41e10da56e9b6857487fb3e409164/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:09,946 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,946 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,947 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,948 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,949 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,949 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,950 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:14:09,952 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,955 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:14:09,956 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0ef41e10da56e9b6857487fb3e409164; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72036239, jitterRate=0.0734236091375351}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:14:09,956 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:09,957 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0ef41e10da56e9b6857487fb3e409164: Running coprocessor pre-open hook at 1733692449930Writing region info on filesystem at 1733692449930Initializing all the Stores at 1733692449933 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692449933Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692449933Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692449933Cleaning up temporary data from old regions at 1733692449949 (+16 ms)Running coprocessor post-open hooks at 1733692449956 (+7 ms)Region opened successfully at 1733692449957 (+1 ms) 2024-12-08T21:14:09,959 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., pid=6, masterSystemTime=1733692449920 2024-12-08T21:14:09,962 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,962 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:09,963 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0ef41e10da56e9b6857487fb3e409164, regionState=OPEN, openSeqNum=2, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:09,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:14:09,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-08T21:14:09,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 in 203 msec 2024-12-08T21:14:09,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-08T21:14:09,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, ASSIGN in 372 msec 2024-12-08T21:14:09,978 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:14:09,978 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692449978"}]},"ts":"1733692449978"} 2024-12-08T21:14:09,982 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T21:14:09,984 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:14:09,988 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-12-08T21:14:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-08T21:14:10,902 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:10,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,912 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cdc6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1806c24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,917 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,921 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:10,923 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:14:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e653e208f502,45981,1733692443391 2024-12-08T21:14:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78210a9b 2024-12-08T21:14:10,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:14:10,930 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:14:10,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,936 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,937 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,942 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6ea95c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,952 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,955 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,959 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e341cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,962 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fba9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,964 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5904dd5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,967 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,967 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,968 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,970 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129161b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,974 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20584f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,976 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7aa9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,979 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c287047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:10,984 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:10,985 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311f9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:10,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:10,987 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:10,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:10,993 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,993 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,993 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,993 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,993 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,994 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,994 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,994 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,993 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:10,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:11,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:11,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:11,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:11,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,010 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-08T21:14:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:14:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:14:11,020 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:11,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:11,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:11,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T21:14:11,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:11,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:11,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:11,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:14:11,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/59a76afd5cab4d3395810577f611f2b1 is 50, key is test_row_0/A:col10/1733692451082/Put/seqid=0 2024-12-08T21:14:11,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741837_1013 (size=12001) 2024-12-08T21:14:11,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692511222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692511224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692511232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692511236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692511236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:14:11,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:11,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:11,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692511381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692511382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692511382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692511383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692511384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692511599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692511600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692511600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692511600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692511600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/59a76afd5cab4d3395810577f611f2b1 2024-12-08T21:14:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:14:11,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:11,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:11,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/e3a405c00c4640b6b4037a51e6abacc7 is 50, key is test_row_0/B:col10/1733692451082/Put/seqid=0 2024-12-08T21:14:11,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741838_1014 (size=12001) 2024-12-08T21:14:11,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/e3a405c00c4640b6b4037a51e6abacc7 2024-12-08T21:14:11,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/48be02c5443147259ae6b60d3a03b844 is 50, key is test_row_0/C:col10/1733692451082/Put/seqid=0 2024-12-08T21:14:11,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741839_1015 (size=12001) 2024-12-08T21:14:11,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/48be02c5443147259ae6b60d3a03b844 2024-12-08T21:14:11,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/59a76afd5cab4d3395810577f611f2b1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/59a76afd5cab4d3395810577f611f2b1 2024-12-08T21:14:11,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:11,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:11,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/59a76afd5cab4d3395810577f611f2b1, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T21:14:11,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/e3a405c00c4640b6b4037a51e6abacc7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e3a405c00c4640b6b4037a51e6abacc7 2024-12-08T21:14:11,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e3a405c00c4640b6b4037a51e6abacc7, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T21:14:11,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/48be02c5443147259ae6b60d3a03b844 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/48be02c5443147259ae6b60d3a03b844 2024-12-08T21:14:11,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/48be02c5443147259ae6b60d3a03b844, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T21:14:11,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0ef41e10da56e9b6857487fb3e409164 in 816ms, sequenceid=13, compaction requested=false 2024-12-08T21:14:11,905 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T21:14:11,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:11,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T21:14:11,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:11,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:11,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:11,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:11,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:11,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:11,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/daccc3fbd4654b59937c0a1fcff1109b is 50, key is test_row_0/A:col10/1733692451210/Put/seqid=0 2024-12-08T21:14:11,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692511959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692511965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:11,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741840_1016 (size=16681) 2024-12-08T21:14:11,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/daccc3fbd4654b59937c0a1fcff1109b 2024-12-08T21:14:11,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:11,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692511976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692511976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692511981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/92c944acd26c41e88d82f439e656e04a is 50, key is test_row_0/B:col10/1733692451210/Put/seqid=0 2024-12-08T21:14:12,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741841_1017 (size=12001) 2024-12-08T21:14:12,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692512089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692512092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692512110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692512118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692512119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:14:12,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692512299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692512301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:12,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:12,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692512319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692512328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692512329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/92c944acd26c41e88d82f439e656e04a 2024-12-08T21:14:12,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/99cb931d59af458f9f64cc562c6f5077 is 50, key is test_row_0/C:col10/1733692451210/Put/seqid=0 2024-12-08T21:14:12,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:12,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741842_1018 (size=12001) 2024-12-08T21:14:12,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/99cb931d59af458f9f64cc562c6f5077 2024-12-08T21:14:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/daccc3fbd4654b59937c0a1fcff1109b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/daccc3fbd4654b59937c0a1fcff1109b 2024-12-08T21:14:12,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/daccc3fbd4654b59937c0a1fcff1109b, entries=250, sequenceid=35, filesize=16.3 K 2024-12-08T21:14:12,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/92c944acd26c41e88d82f439e656e04a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/92c944acd26c41e88d82f439e656e04a 2024-12-08T21:14:12,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/92c944acd26c41e88d82f439e656e04a, entries=150, sequenceid=35, filesize=11.7 K 2024-12-08T21:14:12,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/99cb931d59af458f9f64cc562c6f5077 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/99cb931d59af458f9f64cc562c6f5077 2024-12-08T21:14:12,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/99cb931d59af458f9f64cc562c6f5077, entries=150, sequenceid=35, filesize=11.7 K 2024-12-08T21:14:12,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 0ef41e10da56e9b6857487fb3e409164 in 635ms, sequenceid=35, compaction requested=false 2024-12-08T21:14:12,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:12,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T21:14:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:12,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:12,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:12,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5b9cd8e80b5c4087ab0e7f369f2ddff9 is 50, key is test_row_0/A:col10/1733692452611/Put/seqid=0 2024-12-08T21:14:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741843_1019 (size=14341) 2024-12-08T21:14:12,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5b9cd8e80b5c4087ab0e7f369f2ddff9 2024-12-08T21:14:12,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692512665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692512669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692512673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692512678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692512681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/6f2eef8893b341e59e43bd3d261437b7 is 50, key is test_row_0/B:col10/1733692452611/Put/seqid=0 2024-12-08T21:14:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741844_1020 (size=12001) 2024-12-08T21:14:12,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/6f2eef8893b341e59e43bd3d261437b7 2024-12-08T21:14:12,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/0dcabe26512c4a5ea42f50fbbff0892b is 50, key is test_row_0/C:col10/1733692452611/Put/seqid=0 2024-12-08T21:14:12,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741845_1021 (size=12001) 2024-12-08T21:14:12,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/0dcabe26512c4a5ea42f50fbbff0892b 2024-12-08T21:14:12,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5b9cd8e80b5c4087ab0e7f369f2ddff9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5b9cd8e80b5c4087ab0e7f369f2ddff9 2024-12-08T21:14:12,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5b9cd8e80b5c4087ab0e7f369f2ddff9, entries=200, sequenceid=47, filesize=14.0 K 2024-12-08T21:14:12,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/6f2eef8893b341e59e43bd3d261437b7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6f2eef8893b341e59e43bd3d261437b7 2024-12-08T21:14:12,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:12,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:12,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:12,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692512790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692512790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692512790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692512803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6f2eef8893b341e59e43bd3d261437b7, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T21:14:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692512803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:12,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:12,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/0dcabe26512c4a5ea42f50fbbff0892b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/0dcabe26512c4a5ea42f50fbbff0892b 2024-12-08T21:14:12,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/0dcabe26512c4a5ea42f50fbbff0892b, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T21:14:12,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0ef41e10da56e9b6857487fb3e409164 in 215ms, sequenceid=47, compaction requested=true 2024-12-08T21:14:12,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:12,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:12,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:12,834 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:12,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:12,834 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:12,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:12,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:12,838 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:12,840 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:12,840 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,841 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e3a405c00c4640b6b4037a51e6abacc7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/92c944acd26c41e88d82f439e656e04a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6f2eef8893b341e59e43bd3d261437b7] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=35.2 K 2024-12-08T21:14:12,842 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:12,842 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:12,842 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,842 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/59a76afd5cab4d3395810577f611f2b1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/daccc3fbd4654b59937c0a1fcff1109b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5b9cd8e80b5c4087ab0e7f369f2ddff9] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=42.0 K 2024-12-08T21:14:12,843 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a405c00c4640b6b4037a51e6abacc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733692451017 2024-12-08T21:14:12,844 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c944acd26c41e88d82f439e656e04a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733692451210 2024-12-08T21:14:12,844 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a76afd5cab4d3395810577f611f2b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733692451017 2024-12-08T21:14:12,845 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting daccc3fbd4654b59937c0a1fcff1109b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733692451209 2024-12-08T21:14:12,846 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f2eef8893b341e59e43bd3d261437b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692451943 2024-12-08T21:14:12,846 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b9cd8e80b5c4087ab0e7f369f2ddff9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692451943 2024-12-08T21:14:12,881 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#9 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:12,882 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2bf895c68a854b868025b4d8320e9a64 is 50, key is test_row_0/A:col10/1733692452611/Put/seqid=0 2024-12-08T21:14:12,890 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:12,891 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/a5a2c1f914634e38a2cd05f3a228e4a9 is 50, key is test_row_0/B:col10/1733692452611/Put/seqid=0 2024-12-08T21:14:12,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741846_1022 (size=12104) 2024-12-08T21:14:12,928 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2bf895c68a854b868025b4d8320e9a64 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2bf895c68a854b868025b4d8320e9a64 2024-12-08T21:14:12,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741847_1023 (size=12104) 2024-12-08T21:14:12,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-08T21:14:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,951 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:12,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:12,957 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 2bf895c68a854b868025b4d8320e9a64(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:12,957 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:12,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2353fc04ca5446d4bc83edd58f8dbe29 is 50, key is test_row_0/A:col10/1733692452653/Put/seqid=0 2024-12-08T21:14:12,963 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=13, startTime=1733692452831; duration=0sec 2024-12-08T21:14:12,963 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:12,964 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:12,964 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:12,966 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:12,967 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:12,967 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:12,967 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/48be02c5443147259ae6b60d3a03b844, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/99cb931d59af458f9f64cc562c6f5077, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/0dcabe26512c4a5ea42f50fbbff0892b] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=35.2 K 2024-12-08T21:14:12,968 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48be02c5443147259ae6b60d3a03b844, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733692451017 2024-12-08T21:14:12,972 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99cb931d59af458f9f64cc562c6f5077, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1733692451210 2024-12-08T21:14:12,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741848_1024 (size=12001) 2024-12-08T21:14:12,980 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dcabe26512c4a5ea42f50fbbff0892b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692451943 2024-12-08T21:14:12,981 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2353fc04ca5446d4bc83edd58f8dbe29 2024-12-08T21:14:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9da1fe5c06054239ade552accc6b6529 is 50, key is test_row_0/B:col10/1733692452653/Put/seqid=0 2024-12-08T21:14:13,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:13,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:13,025 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:13,026 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/241bec018e3b4f4f9fc468c1a7d64b0a is 50, key is test_row_0/C:col10/1733692452611/Put/seqid=0 2024-12-08T21:14:13,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741849_1025 (size=12001) 2024-12-08T21:14:13,039 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9da1fe5c06054239ade552accc6b6529 2024-12-08T21:14:13,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692513034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692513038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692513040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692513042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692513044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741850_1026 (size=12104) 2024-12-08T21:14:13,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a9046c9334eb4d86940a0f626e98987b is 50, key is test_row_0/C:col10/1733692452653/Put/seqid=0 2024-12-08T21:14:13,081 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/241bec018e3b4f4f9fc468c1a7d64b0a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/241bec018e3b4f4f9fc468c1a7d64b0a 2024-12-08T21:14:13,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741851_1027 (size=12001) 2024-12-08T21:14:13,116 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a9046c9334eb4d86940a0f626e98987b 2024-12-08T21:14:13,123 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into 241bec018e3b4f4f9fc468c1a7d64b0a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:13,125 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:13,125 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692452834; duration=0sec 2024-12-08T21:14:13,125 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:13,125 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:13,131 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-08T21:14:13,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2353fc04ca5446d4bc83edd58f8dbe29 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2353fc04ca5446d4bc83edd58f8dbe29 2024-12-08T21:14:13,148 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2353fc04ca5446d4bc83edd58f8dbe29, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T21:14:13,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9da1fe5c06054239ade552accc6b6529 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9da1fe5c06054239ade552accc6b6529 2024-12-08T21:14:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692513158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692513158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692513160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:14:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692513160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692513160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,173 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9da1fe5c06054239ade552accc6b6529, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T21:14:13,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a9046c9334eb4d86940a0f626e98987b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a9046c9334eb4d86940a0f626e98987b 2024-12-08T21:14:13,194 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a9046c9334eb4d86940a0f626e98987b, entries=150, sequenceid=68, filesize=11.7 K 2024-12-08T21:14:13,198 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0ef41e10da56e9b6857487fb3e409164 in 247ms, sequenceid=68, compaction requested=false 2024-12-08T21:14:13,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:13,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-08T21:14:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-08T21:14:13,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-08T21:14:13,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1770 sec 2024-12-08T21:14:13,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2030 sec 2024-12-08T21:14:13,364 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/a5a2c1f914634e38a2cd05f3a228e4a9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a5a2c1f914634e38a2cd05f3a228e4a9 2024-12-08T21:14:13,385 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into a5a2c1f914634e38a2cd05f3a228e4a9(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:13,385 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:13,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:14:13,385 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692452834; duration=0sec 2024-12-08T21:14:13,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:13,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:13,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:13,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:13,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:13,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:13,389 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:13,389 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:13,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/926fedab226a41a5accb29ef3576bfff is 50, key is test_row_0/A:col10/1733692453041/Put/seqid=0 2024-12-08T21:14:13,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741852_1028 (size=12001) 2024-12-08T21:14:13,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/926fedab226a41a5accb29ef3576bfff 2024-12-08T21:14:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692513481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692513482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692513492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692513501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692513505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/b783423c6e554df3af4e007aa96c055c is 50, key is test_row_0/B:col10/1733692453041/Put/seqid=0 2024-12-08T21:14:13,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741853_1029 (size=12001) 2024-12-08T21:14:13,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/b783423c6e554df3af4e007aa96c055c 2024-12-08T21:14:13,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c181926030a241f7bfb009b0cdd01fba is 50, key is test_row_0/C:col10/1733692453041/Put/seqid=0 2024-12-08T21:14:13,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741854_1030 (size=12001) 2024-12-08T21:14:13,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692513609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692513609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692513610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692513618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692513621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692513818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692513818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692513828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,840 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:14:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692513839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692513839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:13,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:13,946 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-08T21:14:13,946 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-08T21:14:13,948 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T21:14:13,949 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-08T21:14:13,949 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-08T21:14:13,949 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-08T21:14:13,951 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-08T21:14:13,951 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-08T21:14:14,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c181926030a241f7bfb009b0cdd01fba 2024-12-08T21:14:14,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/926fedab226a41a5accb29ef3576bfff as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/926fedab226a41a5accb29ef3576bfff 2024-12-08T21:14:14,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/926fedab226a41a5accb29ef3576bfff, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T21:14:14,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/b783423c6e554df3af4e007aa96c055c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b783423c6e554df3af4e007aa96c055c 2024-12-08T21:14:14,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b783423c6e554df3af4e007aa96c055c, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T21:14:14,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c181926030a241f7bfb009b0cdd01fba as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c181926030a241f7bfb009b0cdd01fba 2024-12-08T21:14:14,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c181926030a241f7bfb009b0cdd01fba, entries=150, sequenceid=83, filesize=11.7 K 2024-12-08T21:14:14,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 0ef41e10da56e9b6857487fb3e409164 in 735ms, sequenceid=83, compaction requested=true 2024-12-08T21:14:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:14,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:14,122 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:14,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:14,122 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:14,122 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:14,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:14,124 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:14,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:14,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:14,124 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:14,124 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:14,125 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2bf895c68a854b868025b4d8320e9a64, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2353fc04ca5446d4bc83edd58f8dbe29, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/926fedab226a41a5accb29ef3576bfff] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=35.3 K 2024-12-08T21:14:14,126 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf895c68a854b868025b4d8320e9a64, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692451943 2024-12-08T21:14:14,131 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:14,131 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 2353fc04ca5446d4bc83edd58f8dbe29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733692452653 2024-12-08T21:14:14,131 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:14,131 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:14,131 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a5a2c1f914634e38a2cd05f3a228e4a9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9da1fe5c06054239ade552accc6b6529, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b783423c6e554df3af4e007aa96c055c] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=35.3 K 2024-12-08T21:14:14,132 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 926fedab226a41a5accb29ef3576bfff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733692453019 2024-12-08T21:14:14,132 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5a2c1f914634e38a2cd05f3a228e4a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692451943 2024-12-08T21:14:14,136 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9da1fe5c06054239ade552accc6b6529, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733692452653 2024-12-08T21:14:14,137 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b783423c6e554df3af4e007aa96c055c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733692453019 2024-12-08T21:14:14,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692453410, max timestamp=1733692454129Num uniques 859; 2024-12-08T21:14:14,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:14,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-08T21:14:14,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692453410, max timestamp=1733692454129Num uniques 859; 2024-12-08T21:14:14,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:14,159 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:14,160 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/3927a2c58699470783b03e4fda080510 is 50, key is test_row_0/B:col10/1733692453041/Put/seqid=0 2024-12-08T21:14:14,164 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:14,165 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/8914968279b1401985c14c76b1b47da7 is 50, key is test_row_0/A:col10/1733692453041/Put/seqid=0 2024-12-08T21:14:14,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:14,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733692453410, max timestamp=1733692454133Num uniques 859; 2024-12-08T21:14:14,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:14,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:14,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T21:14:14,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:14,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:14,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:14,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741856_1032 (size=12207) 2024-12-08T21:14:14,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/cf3c84d517c14fa98db3b1eaafcf8d1b is 50, key is test_row_0/A:col10/1733692453496/Put/seqid=0 2024-12-08T21:14:14,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741855_1031 (size=12207) 2024-12-08T21:14:14,254 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/8914968279b1401985c14c76b1b47da7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8914968279b1401985c14c76b1b47da7 2024-12-08T21:14:14,273 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 8914968279b1401985c14c76b1b47da7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:14,274 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:14,274 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=13, startTime=1733692454122; duration=0sec 2024-12-08T21:14:14,274 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:14,274 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:14,275 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:14,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741857_1033 (size=12001) 2024-12-08T21:14:14,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/cf3c84d517c14fa98db3b1eaafcf8d1b 2024-12-08T21:14:14,285 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:14,285 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:14,286 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:14,286 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/241bec018e3b4f4f9fc468c1a7d64b0a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a9046c9334eb4d86940a0f626e98987b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c181926030a241f7bfb009b0cdd01fba] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=35.3 K 2024-12-08T21:14:14,287 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 241bec018e3b4f4f9fc468c1a7d64b0a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692451943 2024-12-08T21:14:14,289 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a9046c9334eb4d86940a0f626e98987b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733692452653 2024-12-08T21:14:14,291 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c181926030a241f7bfb009b0cdd01fba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733692453019 2024-12-08T21:14:14,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692454259, max timestamp=1733692454281Num uniques 859; 2024-12-08T21:14:14,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:14,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733692454259, max timestamp=1733692454281Num uniques 859; 2024-12-08T21:14:14,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T21:14:14,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692454259, max timestamp=1733692454288Num uniques 859; 2024-12-08T21:14:14,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:14,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/f94d84f050e24d7d87dacdd0692767f4 is 50, key is test_row_0/B:col10/1733692453496/Put/seqid=0 2024-12-08T21:14:14,343 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#22 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:14,344 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fe25c52d5b444bbe959e791ca5196528 is 50, key is test_row_0/C:col10/1733692453041/Put/seqid=0 2024-12-08T21:14:14,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741858_1034 (size=12001) 2024-12-08T21:14:14,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/f94d84f050e24d7d87dacdd0692767f4 2024-12-08T21:14:14,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692454338, max timestamp=1733692454376Num uniques 1009; 2024-12-08T21:14:14,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-12-08T21:14:14,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741859_1035 (size=12207) 2024-12-08T21:14:14,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692454338, max timestamp=1733692454376Num uniques 1018; 2024-12-08T21:14:14,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T21:14:14,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/42c6bc4d268b42dfa0be082336586a36 is 50, key is test_row_0/C:col10/1733692453496/Put/seqid=0 2024-12-08T21:14:14,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692454338, max timestamp=1733692454376Num uniques 1018; 2024-12-08T21:14:14,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T21:14:14,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741860_1036 (size=12001) 2024-12-08T21:14:14,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733692454418, max timestamp=1733692454435Num uniques 1077; 2024-12-08T21:14:14,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733692454418, max timestamp=1733692454435Num uniques 1077; 2024-12-08T21:14:14,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-08T21:14:14,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-08T21:14:14,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733692454418, max timestamp=1733692454435Num uniques 1077; 2024-12-08T21:14:14,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-08T21:14:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692514431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692514495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10796, totalHeapSize=24784, min timestamp=1733692454477, max timestamp=1733692454499Num uniques 1086; 2024-12-08T21:14:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49726, new segment data size=10796, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-08T21:14:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692454477, max timestamp=1733692454499Num uniques 1086; 2024-12-08T21:14:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-08T21:14:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692454477, max timestamp=1733692454499Num uniques 1086; 2024-12-08T21:14:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-08T21:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733692454584, max timestamp=1733692454601Num uniques 1095; 2024-12-08T21:14:14,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-08T21:14:14,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733692454584, max timestamp=1733692454601Num uniques 1095; 2024-12-08T21:14:14,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-08T21:14:14,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733692454584, max timestamp=1733692454601Num uniques 1095; 2024-12-08T21:14:14,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-12-08T21:14:14,654 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/3927a2c58699470783b03e4fda080510 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3927a2c58699470783b03e4fda080510 2024-12-08T21:14:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692514646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692514649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692514652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692514653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692514656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,678 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into 3927a2c58699470783b03e4fda080510(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:14,678 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:14,678 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692454122; duration=0sec 2024-12-08T21:14:14,678 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:14,678 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692514768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692514768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692514768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692514768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692514779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:14,794 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fe25c52d5b444bbe959e791ca5196528 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fe25c52d5b444bbe959e791ca5196528 2024-12-08T21:14:14,807 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into fe25c52d5b444bbe959e791ca5196528(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:14,807 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:14,808 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692454124; duration=0sec 2024-12-08T21:14:14,808 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:14,808 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:14,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/42c6bc4d268b42dfa0be082336586a36 2024-12-08T21:14:14,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/cf3c84d517c14fa98db3b1eaafcf8d1b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/cf3c84d517c14fa98db3b1eaafcf8d1b 2024-12-08T21:14:14,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/cf3c84d517c14fa98db3b1eaafcf8d1b, entries=150, sequenceid=106, filesize=11.7 K 2024-12-08T21:14:14,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/f94d84f050e24d7d87dacdd0692767f4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/f94d84f050e24d7d87dacdd0692767f4 2024-12-08T21:14:14,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/f94d84f050e24d7d87dacdd0692767f4, entries=150, sequenceid=106, filesize=11.7 K 2024-12-08T21:14:14,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/42c6bc4d268b42dfa0be082336586a36 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42c6bc4d268b42dfa0be082336586a36 2024-12-08T21:14:14,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42c6bc4d268b42dfa0be082336586a36, entries=150, sequenceid=106, filesize=11.7 K 2024-12-08T21:14:14,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.86 KB/40815, heapSize ~105.12 KB/107640, currentSize=147.60 KB/151140 for 0ef41e10da56e9b6857487fb3e409164 in 723ms, sequenceid=106, compaction requested=false 2024-12-08T21:14:14,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:14,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=161.02 KB heapSize=452.63 KB 2024-12-08T21:14:14,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733692454644, max timestamp=1733692454980Num uniques 1154; 2024-12-08T21:14:14,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:14,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733692454644, max timestamp=1733692454980Num uniques 1154; 2024-12-08T21:14:14,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=6870, suffix heap size=147984,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=150 2024-12-08T21:14:14,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:14,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52844, new segment data size=6870, suffix heap size=147984,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=150 2024-12-08T21:14:14,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:14,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:14,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:14,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:14:14,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5e7f51894740495ca78bc00d93147395 is 50, key is test_row_0/A:col10/1733692454979/Put/seqid=0 2024-12-08T21:14:15,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741861_1037 (size=14541) 2024-12-08T21:14:15,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692515009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692515011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692515011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692515017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692515017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692515128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692515128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692515128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692515128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692515138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-08T21:14:15,189 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:15,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-08T21:14:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T21:14:15,192 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:15,193 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:15,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T21:14:15,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692515338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692515338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692515338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692515338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:15,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692515349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.01 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5e7f51894740495ca78bc00d93147395 2024-12-08T21:14:15,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/5d16c4d639d147c1b07df3e5c1fbcd2f is 50, key is test_row_0/B:col10/1733692454979/Put/seqid=0 2024-12-08T21:14:15,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741862_1038 (size=12151) 2024-12-08T21:14:15,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.01 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/5d16c4d639d147c1b07df3e5c1fbcd2f 2024-12-08T21:14:15,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fb9244a840fc41b5b27fece4fe75f340 is 50, key is test_row_0/C:col10/1733692454979/Put/seqid=0 2024-12-08T21:14:15,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:15,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T21:14:15,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741863_1039 (size=12151) 2024-12-08T21:14:15,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fb9244a840fc41b5b27fece4fe75f340 2024-12-08T21:14:15,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5e7f51894740495ca78bc00d93147395 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5e7f51894740495ca78bc00d93147395 2024-12-08T21:14:15,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5e7f51894740495ca78bc00d93147395, entries=200, sequenceid=217, filesize=14.2 K 2024-12-08T21:14:15,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/5d16c4d639d147c1b07df3e5c1fbcd2f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/5d16c4d639d147c1b07df3e5c1fbcd2f 2024-12-08T21:14:15,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/5d16c4d639d147c1b07df3e5c1fbcd2f, entries=150, sequenceid=217, filesize=11.9 K 2024-12-08T21:14:15,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fb9244a840fc41b5b27fece4fe75f340 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fb9244a840fc41b5b27fece4fe75f340 2024-12-08T21:14:15,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fb9244a840fc41b5b27fece4fe75f340, entries=150, sequenceid=217, filesize=11.9 K 2024-12-08T21:14:15,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~77.93 KB/79802, heapSize ~214.41 KB/219552, currentSize=100.63 KB/103050 for 0ef41e10da56e9b6857487fb3e409164 in 593ms, sequenceid=217, compaction requested=true 2024-12-08T21:14:15,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:15,575 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:15,575 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:15,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:15,576 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:15,577 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:15,577 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,577 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3927a2c58699470783b03e4fda080510, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/f94d84f050e24d7d87dacdd0692767f4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/5d16c4d639d147c1b07df3e5c1fbcd2f] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=35.5 K 2024-12-08T21:14:15,577 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:15,578 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:15,578 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,578 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8914968279b1401985c14c76b1b47da7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/cf3c84d517c14fa98db3b1eaafcf8d1b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5e7f51894740495ca78bc00d93147395] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=37.8 K 2024-12-08T21:14:15,580 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8914968279b1401985c14c76b1b47da7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733692453019 2024-12-08T21:14:15,580 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 3927a2c58699470783b03e4fda080510, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733692453019 2024-12-08T21:14:15,581 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf3c84d517c14fa98db3b1eaafcf8d1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733692453496 2024-12-08T21:14:15,581 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f94d84f050e24d7d87dacdd0692767f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733692453496 2024-12-08T21:14:15,582 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d16c4d639d147c1b07df3e5c1fbcd2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733692454649 2024-12-08T21:14:15,582 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e7f51894740495ca78bc00d93147395, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733692454649 2024-12-08T21:14:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:15,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:15,600 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:15,601 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:15,601 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/2bd96241a38f42fea23ea0f6d66e7e19 is 50, key is test_row_0/B:col10/1733692454979/Put/seqid=0 2024-12-08T21:14:15,601 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/63bb382a14864b0ea70931a63b0b742a is 50, key is test_row_0/A:col10/1733692454979/Put/seqid=0 2024-12-08T21:14:15,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741865_1041 (size=12459) 2024-12-08T21:14:15,639 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/2bd96241a38f42fea23ea0f6d66e7e19 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2bd96241a38f42fea23ea0f6d66e7e19 2024-12-08T21:14:15,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741864_1040 (size=12459) 2024-12-08T21:14:15,655 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/63bb382a14864b0ea70931a63b0b742a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/63bb382a14864b0ea70931a63b0b742a 2024-12-08T21:14:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:15,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T21:14:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:15,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:15,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:15,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,660 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into 2bd96241a38f42fea23ea0f6d66e7e19(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:15,661 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:15,661 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692455575; duration=0sec 2024-12-08T21:14:15,661 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:15,661 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:15,661 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e71346c764fd48aa9b9511361111a672 is 50, key is test_row_0/A:col10/1733692455009/Put/seqid=0 2024-12-08T21:14:15,668 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:15,669 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 63bb382a14864b0ea70931a63b0b742a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:15,669 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:15,669 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=13, startTime=1733692455574; duration=0sec 2024-12-08T21:14:15,669 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:15,669 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:15,669 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:15,669 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,670 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fe25c52d5b444bbe959e791ca5196528, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42c6bc4d268b42dfa0be082336586a36, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fb9244a840fc41b5b27fece4fe75f340] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=35.5 K 2024-12-08T21:14:15,671 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting fe25c52d5b444bbe959e791ca5196528, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733692453019 2024-12-08T21:14:15,672 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 42c6bc4d268b42dfa0be082336586a36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1733692453496 2024-12-08T21:14:15,673 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9244a840fc41b5b27fece4fe75f340, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733692454649 2024-12-08T21:14:15,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692515685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692515686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692515688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692515688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692515690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741866_1042 (size=14541) 2024-12-08T21:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e71346c764fd48aa9b9511361111a672 2024-12-08T21:14:15,705 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#30 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:15,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e3f2b8bafaf74f42a0870d33407c93b3 is 50, key is test_row_0/C:col10/1733692454979/Put/seqid=0 2024-12-08T21:14:15,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/a9b2f592e68a4ff5a94dfa238c927b9c is 50, key is test_row_0/B:col10/1733692455009/Put/seqid=0 2024-12-08T21:14:15,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741867_1043 (size=12459) 2024-12-08T21:14:15,744 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e3f2b8bafaf74f42a0870d33407c93b3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3f2b8bafaf74f42a0870d33407c93b3 2024-12-08T21:14:15,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741868_1044 (size=12151) 2024-12-08T21:14:15,758 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into e3f2b8bafaf74f42a0870d33407c93b3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:15,758 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:15,758 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692455583; duration=0sec 2024-12-08T21:14:15,759 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:15,759 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:15,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692515798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692515799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692515799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692515799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692515799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:15,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:15,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:15,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:15,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T21:14:15,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:15,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:15,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:15,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:15,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692516008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692516008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692516009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692516009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692516008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/a9b2f592e68a4ff5a94dfa238c927b9c 2024-12-08T21:14:16,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/64cbafc999694111939b183b3dfe23f2 is 50, key is test_row_0/C:col10/1733692455009/Put/seqid=0 2024-12-08T21:14:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741869_1045 (size=12151) 2024-12-08T21:14:16,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:16,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692516318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692516318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692516319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692516319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T21:14:16,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692516328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:16,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:16,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/64cbafc999694111939b183b3dfe23f2 2024-12-08T21:14:16,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e71346c764fd48aa9b9511361111a672 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e71346c764fd48aa9b9511361111a672 2024-12-08T21:14:16,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:16,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:16,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e71346c764fd48aa9b9511361111a672, entries=200, sequenceid=236, filesize=14.2 K 2024-12-08T21:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:16,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/a9b2f592e68a4ff5a94dfa238c927b9c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a9b2f592e68a4ff5a94dfa238c927b9c 2024-12-08T21:14:16,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a9b2f592e68a4ff5a94dfa238c927b9c, entries=150, sequenceid=236, filesize=11.9 K 2024-12-08T21:14:16,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/64cbafc999694111939b183b3dfe23f2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/64cbafc999694111939b183b3dfe23f2 2024-12-08T21:14:16,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/64cbafc999694111939b183b3dfe23f2, entries=150, sequenceid=236, filesize=11.9 K 2024-12-08T21:14:16,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0ef41e10da56e9b6857487fb3e409164 in 969ms, sequenceid=236, compaction requested=false 2024-12-08T21:14:16,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:16,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-08T21:14:16,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,748 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:16,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e87f53e440194209b586ef0d46fbe71e is 50, key is test_row_0/A:col10/1733692455683/Put/seqid=0 2024-12-08T21:14:16,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741870_1046 (size=12151) 2024-12-08T21:14:16,781 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e87f53e440194209b586ef0d46fbe71e 2024-12-08T21:14:16,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/409aafd0a8c24fa78563eaaaef1941aa is 50, key is test_row_0/B:col10/1733692455683/Put/seqid=0 2024-12-08T21:14:16,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741871_1047 (size=12151) 2024-12-08T21:14:16,811 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/409aafd0a8c24fa78563eaaaef1941aa 2024-12-08T21:14:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/cf3f80aadda646368129d282f393297c is 50, key is test_row_0/C:col10/1733692455683/Put/seqid=0 2024-12-08T21:14:16,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:16,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741872_1048 (size=12151) 2024-12-08T21:14:16,841 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/cf3f80aadda646368129d282f393297c 2024-12-08T21:14:16,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e87f53e440194209b586ef0d46fbe71e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e87f53e440194209b586ef0d46fbe71e 2024-12-08T21:14:16,864 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e87f53e440194209b586ef0d46fbe71e, entries=150, sequenceid=252, filesize=11.9 K 2024-12-08T21:14:16,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/409aafd0a8c24fa78563eaaaef1941aa as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/409aafd0a8c24fa78563eaaaef1941aa 2024-12-08T21:14:16,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692456854, max timestamp=1733692456866Num uniques 859; 2024-12-08T21:14:16,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:16,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692456854, max timestamp=1733692456866Num uniques 859; 2024-12-08T21:14:16,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:16,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692456854, max timestamp=1733692456866Num uniques 859; 2024-12-08T21:14:16,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:16,884 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/409aafd0a8c24fa78563eaaaef1941aa, entries=150, sequenceid=252, filesize=11.9 K 2024-12-08T21:14:16,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/cf3f80aadda646368129d282f393297c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/cf3f80aadda646368129d282f393297c 2024-12-08T21:14:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692516889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692516893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692516895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,904 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/cf3f80aadda646368129d282f393297c, entries=150, sequenceid=252, filesize=11.9 K 2024-12-08T21:14:16,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692516898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:16,908 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 0ef41e10da56e9b6857487fb3e409164 in 159ms, sequenceid=252, compaction requested=true 2024-12-08T21:14:16,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:16,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:16,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-08T21:14:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-08T21:14:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:16,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-08T21:14:16,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:16,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:16,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-08T21:14:16,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-12-08T21:14:16,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/b9daa572965a449d8a62a0a359d48925 is 50, key is test_row_0/A:col10/1733692456910/Put/seqid=0 2024-12-08T21:14:16,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7300 sec 2024-12-08T21:14:16,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741873_1049 (size=14741) 2024-12-08T21:14:16,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/b9daa572965a449d8a62a0a359d48925 2024-12-08T21:14:16,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/6b4f69354fb24651bc933be3b39caa84 is 50, key is test_row_0/B:col10/1733692456910/Put/seqid=0 2024-12-08T21:14:16,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741874_1050 (size=12301) 2024-12-08T21:14:16,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692516984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692517009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692517009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692517009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692517009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692517098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692517229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692517229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692517229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692517233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692517308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-08T21:14:17,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:17,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-08T21:14:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T21:14:17,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:17,342 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:17,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:17,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/6b4f69354fb24651bc933be3b39caa84 2024-12-08T21:14:17,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a76e02fcd94543bbb57766ca48ff204c is 50, key is test_row_0/C:col10/1733692456910/Put/seqid=0 2024-12-08T21:14:17,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741875_1051 (size=12301) 2024-12-08T21:14:17,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a76e02fcd94543bbb57766ca48ff204c 2024-12-08T21:14:17,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/b9daa572965a449d8a62a0a359d48925 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b9daa572965a449d8a62a0a359d48925 2024-12-08T21:14:17,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b9daa572965a449d8a62a0a359d48925, entries=200, sequenceid=286, filesize=14.4 K 2024-12-08T21:14:17,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/6b4f69354fb24651bc933be3b39caa84 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6b4f69354fb24651bc933be3b39caa84 2024-12-08T21:14:17,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6b4f69354fb24651bc933be3b39caa84, entries=150, sequenceid=286, filesize=12.0 K 2024-12-08T21:14:17,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a76e02fcd94543bbb57766ca48ff204c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a76e02fcd94543bbb57766ca48ff204c 2024-12-08T21:14:17,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a76e02fcd94543bbb57766ca48ff204c, entries=150, sequenceid=286, filesize=12.0 K 2024-12-08T21:14:17,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 0ef41e10da56e9b6857487fb3e409164 in 524ms, sequenceid=286, compaction requested=true 2024-12-08T21:14:17,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:17,437 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:17,437 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:17,439 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53892 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:17,439 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:17,439 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:17,439 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:17,439 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:17,439 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:17,439 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2bd96241a38f42fea23ea0f6d66e7e19, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a9b2f592e68a4ff5a94dfa238c927b9c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/409aafd0a8c24fa78563eaaaef1941aa, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6b4f69354fb24651bc933be3b39caa84] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=47.9 K 2024-12-08T21:14:17,439 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/63bb382a14864b0ea70931a63b0b742a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e71346c764fd48aa9b9511361111a672, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e87f53e440194209b586ef0d46fbe71e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b9daa572965a449d8a62a0a359d48925] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=52.6 K 2024-12-08T21:14:17,440 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd96241a38f42fea23ea0f6d66e7e19, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733692454649 2024-12-08T21:14:17,440 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63bb382a14864b0ea70931a63b0b742a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733692454649 2024-12-08T21:14:17,440 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b2f592e68a4ff5a94dfa238c927b9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733692455009 2024-12-08T21:14:17,440 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71346c764fd48aa9b9511361111a672, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733692455009 2024-12-08T21:14:17,441 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 409aafd0a8c24fa78563eaaaef1941aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733692455683 2024-12-08T21:14:17,441 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting e87f53e440194209b586ef0d46fbe71e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733692455683 2024-12-08T21:14:17,442 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b4f69354fb24651bc933be3b39caa84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692456885 2024-12-08T21:14:17,443 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9daa572965a449d8a62a0a359d48925, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692456885 2024-12-08T21:14:17,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T21:14:17,476 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:17,477 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/ab66001a4ff247339e58186f536beece is 50, key is test_row_0/A:col10/1733692456910/Put/seqid=0 2024-12-08T21:14:17,487 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:17,487 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/3d86f5c907a4428d81d1907ea9c34926 is 50, key is test_row_0/B:col10/1733692456910/Put/seqid=0 2024-12-08T21:14:17,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-08T21:14:17,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:17,496 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:14:17,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:17,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:17,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741876_1052 (size=12745) 2024-12-08T21:14:17,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741877_1053 (size=12745) 2024-12-08T21:14:17,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/67bce704566c45938730b989a62e9706 is 50, key is test_row_0/A:col10/1733692456974/Put/seqid=0 2024-12-08T21:14:17,523 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/3d86f5c907a4428d81d1907ea9c34926 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3d86f5c907a4428d81d1907ea9c34926 2024-12-08T21:14:17,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:17,552 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into 3d86f5c907a4428d81d1907ea9c34926(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:17,552 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:17,553 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=12, startTime=1733692457437; duration=0sec 2024-12-08T21:14:17,553 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:17,553 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:17,553 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:17,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:17,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741878_1054 (size=12301) 2024-12-08T21:14:17,557 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:17,557 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:17,557 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:17,557 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3f2b8bafaf74f42a0870d33407c93b3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/64cbafc999694111939b183b3dfe23f2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/cf3f80aadda646368129d282f393297c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a76e02fcd94543bbb57766ca48ff204c] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=47.9 K 2024-12-08T21:14:17,558 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f2b8bafaf74f42a0870d33407c93b3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733692454649 2024-12-08T21:14:17,559 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/67bce704566c45938730b989a62e9706 2024-12-08T21:14:17,559 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 64cbafc999694111939b183b3dfe23f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733692455009 2024-12-08T21:14:17,560 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting cf3f80aadda646368129d282f393297c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733692455683 2024-12-08T21:14:17,560 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a76e02fcd94543bbb57766ca48ff204c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692456885 2024-12-08T21:14:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c6e78abf71e94d26b23deb00eb836568 is 50, key is test_row_0/B:col10/1733692456974/Put/seqid=0 2024-12-08T21:14:17,580 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:17,581 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e3a2246e3ff94497bfaca5a0adf96adf is 50, key is test_row_0/C:col10/1733692456910/Put/seqid=0 2024-12-08T21:14:17,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741879_1055 (size=12301) 2024-12-08T21:14:17,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741880_1056 (size=12745) 2024-12-08T21:14:17,595 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c6e78abf71e94d26b23deb00eb836568 2024-12-08T21:14:17,605 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e3a2246e3ff94497bfaca5a0adf96adf as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a2246e3ff94497bfaca5a0adf96adf 2024-12-08T21:14:17,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/ab9b775c303943b7b06a1cf67515b2e5 is 50, key is test_row_0/C:col10/1733692456974/Put/seqid=0 2024-12-08T21:14:17,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692517610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,617 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into e3a2246e3ff94497bfaca5a0adf96adf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:17,618 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:17,618 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=12, startTime=1733692457437; duration=0sec 2024-12-08T21:14:17,618 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:17,618 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692517612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692517612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692517617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692517618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741881_1057 (size=12301) 2024-12-08T21:14:17,635 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/ab9b775c303943b7b06a1cf67515b2e5 2024-12-08T21:14:17,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/67bce704566c45938730b989a62e9706 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/67bce704566c45938730b989a62e9706 2024-12-08T21:14:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T21:14:17,659 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/67bce704566c45938730b989a62e9706, entries=150, sequenceid=298, filesize=12.0 K 2024-12-08T21:14:17,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c6e78abf71e94d26b23deb00eb836568 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6e78abf71e94d26b23deb00eb836568 2024-12-08T21:14:17,671 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6e78abf71e94d26b23deb00eb836568, entries=150, sequenceid=298, filesize=12.0 K 2024-12-08T21:14:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/ab9b775c303943b7b06a1cf67515b2e5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/ab9b775c303943b7b06a1cf67515b2e5 2024-12-08T21:14:17,684 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/ab9b775c303943b7b06a1cf67515b2e5, entries=150, sequenceid=298, filesize=12.0 K 2024-12-08T21:14:17,685 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0ef41e10da56e9b6857487fb3e409164 in 189ms, sequenceid=298, compaction requested=false 2024-12-08T21:14:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-08T21:14:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-08T21:14:17,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-08T21:14:17,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 345 msec 2024-12-08T21:14:17,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 354 msec 2024-12-08T21:14:17,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692457569, max timestamp=1733692457730Num uniques 859; 2024-12-08T21:14:17,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692457569, max timestamp=1733692457730Num uniques 859; 2024-12-08T21:14:17,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692457569, max timestamp=1733692457730Num uniques 859; 2024-12-08T21:14:17,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:17,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:17,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:17,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-08T21:14:17,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:17,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:17,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:17,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:17,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:17,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:17,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2fa8c58644844dac95b905f2a5d0a451 is 50, key is test_row_0/A:col10/1733692457739/Put/seqid=0 2024-12-08T21:14:17,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741882_1058 (size=12301) 2024-12-08T21:14:17,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2fa8c58644844dac95b905f2a5d0a451 2024-12-08T21:14:17,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1733692457747, max timestamp=1733692457777Num uniques 859; 2024-12-08T21:14:17,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1733692457747, max timestamp=1733692457777Num uniques 859; 2024-12-08T21:14:17,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-12-08T21:14:17,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-12-08T21:14:17,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692457765, max timestamp=1733692457777Num uniques 859; 2024-12-08T21:14:17,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:17,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/89722afe04e4425b9c561462ede3cf8c is 50, key is test_row_0/B:col10/1733692457739/Put/seqid=0 2024-12-08T21:14:17,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692517805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692517806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692517807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692517808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741883_1059 (size=12301) 2024-12-08T21:14:17,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/89722afe04e4425b9c561462ede3cf8c 2024-12-08T21:14:17,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/42d80fcd50534c489df0960e716965a1 is 50, key is test_row_0/C:col10/1733692457739/Put/seqid=0 2024-12-08T21:14:17,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741884_1060 (size=12301) 2024-12-08T21:14:17,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/42d80fcd50534c489df0960e716965a1 2024-12-08T21:14:17,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2fa8c58644844dac95b905f2a5d0a451 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2fa8c58644844dac95b905f2a5d0a451 2024-12-08T21:14:17,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2fa8c58644844dac95b905f2a5d0a451, entries=150, sequenceid=326, filesize=12.0 K 2024-12-08T21:14:17,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/89722afe04e4425b9c561462ede3cf8c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/89722afe04e4425b9c561462ede3cf8c 2024-12-08T21:14:17,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/89722afe04e4425b9c561462ede3cf8c, entries=150, sequenceid=326, filesize=12.0 K 2024-12-08T21:14:17,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/42d80fcd50534c489df0960e716965a1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42d80fcd50534c489df0960e716965a1 2024-12-08T21:14:17,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42d80fcd50534c489df0960e716965a1, entries=150, sequenceid=326, filesize=12.0 K 2024-12-08T21:14:17,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=127.47 KB/130530 for 0ef41e10da56e9b6857487fb3e409164 in 162ms, sequenceid=326, compaction requested=true 2024-12-08T21:14:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:17,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:17,903 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 4 compacting, 2 eligible, 16 blocking 2024-12-08T21:14:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:17,904 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T21:14:17,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T21:14:17,904 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T21:14:17,904 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. because compaction request was cancelled 2024-12-08T21:14:17,904 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:17,904 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:17,908 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:17,908 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:17,908 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:17,909 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3d86f5c907a4428d81d1907ea9c34926, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6e78abf71e94d26b23deb00eb836568, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/89722afe04e4425b9c561462ede3cf8c] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=36.5 K 2024-12-08T21:14:17,909 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d86f5c907a4428d81d1907ea9c34926, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692456885 2024-12-08T21:14:17,910 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e78abf71e94d26b23deb00eb836568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733692456963 2024-12-08T21:14:17,911 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 89722afe04e4425b9c561462ede3cf8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733692457732 2024-12-08T21:14:17,922 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/ab66001a4ff247339e58186f536beece as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/ab66001a4ff247339e58186f536beece 2024-12-08T21:14:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:17,927 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:17,927 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/b17df33404b146d5a2ac33e0b01b0fa7 is 50, key is test_row_0/B:col10/1733692457739/Put/seqid=0 2024-12-08T21:14:17,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-12-08T21:14:17,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:17,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:17,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:17,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:17,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:17,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:17,939 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into ab66001a4ff247339e58186f536beece(size=12.4 K), total size for store is 36.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:17,939 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:17,939 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=12, startTime=1733692457436; duration=0sec 2024-12-08T21:14:17,939 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:17,939 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:17,939 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:17,941 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:17,941 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:17,942 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:17,942 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a2246e3ff94497bfaca5a0adf96adf, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/ab9b775c303943b7b06a1cf67515b2e5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42d80fcd50534c489df0960e716965a1] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=36.5 K 2024-12-08T21:14:17,942 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3a2246e3ff94497bfaca5a0adf96adf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692456885 2024-12-08T21:14:17,943 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9b775c303943b7b06a1cf67515b2e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733692456963 2024-12-08T21:14:17,943 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d80fcd50534c489df0960e716965a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733692457732 2024-12-08T21:14:17,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692517938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/9484f0c43ac8451b8e7d25a999d24559 is 50, key is test_row_0/A:col10/1733692457928/Put/seqid=0 2024-12-08T21:14:17,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692517941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692517944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692517947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:17,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741885_1061 (size=12847) 2024-12-08T21:14:17,962 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:17,963 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8b8a65c949e045e985132665996f3c71 is 50, key is test_row_0/C:col10/1733692457739/Put/seqid=0 2024-12-08T21:14:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-08T21:14:17,969 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:17,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-08T21:14:17,973 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:17,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:17,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:17,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741886_1062 (size=14741) 2024-12-08T21:14:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:14:17,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741887_1063 (size=12847) 2024-12-08T21:14:17,995 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8b8a65c949e045e985132665996f3c71 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8b8a65c949e045e985132665996f3c71 2024-12-08T21:14:18,005 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into 8b8a65c949e045e985132665996f3c71(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:18,005 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:18,005 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692457903; duration=0sec 2024-12-08T21:14:18,006 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,006 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692518058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692518058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692518058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692518058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:14:18,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T21:14:18,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:18,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692518130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692518267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692518268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692518268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692518268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T21:14:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:14:18,368 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/b17df33404b146d5a2ac33e0b01b0fa7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b17df33404b146d5a2ac33e0b01b0fa7 2024-12-08T21:14:18,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/9484f0c43ac8451b8e7d25a999d24559 2024-12-08T21:14:18,382 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into b17df33404b146d5a2ac33e0b01b0fa7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:18,382 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:18,382 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692457903; duration=0sec 2024-12-08T21:14:18,383 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,383 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:18,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/7a30542da1ee4818a1bf542a5c77319d is 50, key is test_row_0/B:col10/1733692457928/Put/seqid=0 2024-12-08T21:14:18,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741888_1064 (size=12301) 2024-12-08T21:14:18,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/7a30542da1ee4818a1bf542a5c77319d 2024-12-08T21:14:18,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/94e57e248fee4cbf8a4ec4c19d9510bc is 50, key is test_row_0/C:col10/1733692457928/Put/seqid=0 2024-12-08T21:14:18,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T21:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741889_1065 (size=12301) 2024-12-08T21:14:18,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/94e57e248fee4cbf8a4ec4c19d9510bc 2024-12-08T21:14:18,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/9484f0c43ac8451b8e7d25a999d24559 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/9484f0c43ac8451b8e7d25a999d24559 2024-12-08T21:14:18,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/9484f0c43ac8451b8e7d25a999d24559, entries=200, sequenceid=362, filesize=14.4 K 2024-12-08T21:14:18,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/7a30542da1ee4818a1bf542a5c77319d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/7a30542da1ee4818a1bf542a5c77319d 2024-12-08T21:14:18,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/7a30542da1ee4818a1bf542a5c77319d, entries=150, sequenceid=362, filesize=12.0 K 2024-12-08T21:14:18,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/94e57e248fee4cbf8a4ec4c19d9510bc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94e57e248fee4cbf8a4ec4c19d9510bc 2024-12-08T21:14:18,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94e57e248fee4cbf8a4ec4c19d9510bc, entries=150, sequenceid=362, filesize=12.0 K 2024-12-08T21:14:18,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for 0ef41e10da56e9b6857487fb3e409164 in 595ms, sequenceid=362, compaction requested=true 2024-12-08T21:14:18,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:18,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:18,524 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:18,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:18,525 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T21:14:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:18,526 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T21:14:18,526 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T21:14:18,526 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. because compaction request was cancelled 2024-12-08T21:14:18,526 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:18,526 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T21:14:18,527 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:18,527 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:18,527 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,527 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/ab66001a4ff247339e58186f536beece, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/67bce704566c45938730b989a62e9706, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2fa8c58644844dac95b905f2a5d0a451, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/9484f0c43ac8451b8e7d25a999d24559] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=50.9 K 2024-12-08T21:14:18,527 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T21:14:18,527 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T21:14:18,528 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. because compaction request was cancelled 2024-12-08T21:14:18,528 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:18,528 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab66001a4ff247339e58186f536beece, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692456885 2024-12-08T21:14:18,528 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67bce704566c45938730b989a62e9706, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733692456963 2024-12-08T21:14:18,529 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa8c58644844dac95b905f2a5d0a451, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733692457732 2024-12-08T21:14:18,529 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9484f0c43ac8451b8e7d25a999d24559, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733692457806 2024-12-08T21:14:18,546 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#53 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:18,546 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/8367fb52fef846919bbb185a5f9a3b66 is 50, key is test_row_0/A:col10/1733692457928/Put/seqid=0 2024-12-08T21:14:18,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741890_1066 (size=12881) 2024-12-08T21:14:18,578 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/8367fb52fef846919bbb185a5f9a3b66 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8367fb52fef846919bbb185a5f9a3b66 2024-12-08T21:14:18,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:14:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:18,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:18,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:18,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:18,594 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 8367fb52fef846919bbb185a5f9a3b66(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:18,594 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:18,594 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=12, startTime=1733692458524; duration=0sec 2024-12-08T21:14:18,594 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,594 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:18,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:14:18,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/91b64a6482674308ae6af99b0c490b78 is 50, key is test_row_0/A:col10/1733692458582/Put/seqid=0 2024-12-08T21:14:18,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741891_1067 (size=12301) 2024-12-08T21:14:18,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/91b64a6482674308ae6af99b0c490b78 2024-12-08T21:14:18,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T21:14:18,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:18,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/29426e9d84ec45658f53f668b8d7158a is 50, key is test_row_0/B:col10/1733692458582/Put/seqid=0 2024-12-08T21:14:18,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692518642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692458624, max timestamp=1733692458652Num uniques 859; 2024-12-08T21:14:18,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692458624, max timestamp=1733692458652Num uniques 859; 2024-12-08T21:14:18,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692458624, max timestamp=1733692458652Num uniques 859; 2024-12-08T21:14:18,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:14:18,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:14:18,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:14:18,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741892_1068 (size=12301) 2024-12-08T21:14:18,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/29426e9d84ec45658f53f668b8d7158a 2024-12-08T21:14:18,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/39a5da8de49f4d7ea4bcf6ad69e12565 is 50, key is test_row_0/C:col10/1733692458582/Put/seqid=0 2024-12-08T21:14:18,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692518701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692518702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692518705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741893_1069 (size=12301) 2024-12-08T21:14:18,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/39a5da8de49f4d7ea4bcf6ad69e12565 2024-12-08T21:14:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/91b64a6482674308ae6af99b0c490b78 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/91b64a6482674308ae6af99b0c490b78 2024-12-08T21:14:18,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/91b64a6482674308ae6af99b0c490b78, entries=150, sequenceid=376, filesize=12.0 K 2024-12-08T21:14:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/29426e9d84ec45658f53f668b8d7158a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/29426e9d84ec45658f53f668b8d7158a 2024-12-08T21:14:18,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/29426e9d84ec45658f53f668b8d7158a, entries=150, sequenceid=376, filesize=12.0 K 2024-12-08T21:14:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/39a5da8de49f4d7ea4bcf6ad69e12565 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/39a5da8de49f4d7ea4bcf6ad69e12565 2024-12-08T21:14:18,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692518758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/39a5da8de49f4d7ea4bcf6ad69e12565, entries=150, sequenceid=376, filesize=12.0 K 2024-12-08T21:14:18,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-08T21:14:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0ef41e10da56e9b6857487fb3e409164 in 200ms, sequenceid=376, compaction requested=true 2024-12-08T21:14:18,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:18,785 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-08T21:14:18,785 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-08T21:14:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:18,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:18,787 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-08T21:14:18,787 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-08T21:14:18,787 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. because compaction request was cancelled 2024-12-08T21:14:18,787 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:18,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 0 2024-12-08T21:14:18,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,789 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:18,791 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:18,791 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:18,791 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,791 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b17df33404b146d5a2ac33e0b01b0fa7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/7a30542da1ee4818a1bf542a5c77319d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/29426e9d84ec45658f53f668b8d7158a] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=36.6 K 2024-12-08T21:14:18,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b17df33404b146d5a2ac33e0b01b0fa7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733692457732 2024-12-08T21:14:18,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:18,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a30542da1ee4818a1bf542a5c77319d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733692457806 2024-12-08T21:14:18,793 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29426e9d84ec45658f53f668b8d7158a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733692457940 2024-12-08T21:14:18,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,796 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:18,798 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:18,798 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:18,798 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:18,799 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8b8a65c949e045e985132665996f3c71, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94e57e248fee4cbf8a4ec4c19d9510bc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/39a5da8de49f4d7ea4bcf6ad69e12565] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=36.6 K 2024-12-08T21:14:18,800 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b8a65c949e045e985132665996f3c71, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733692457732 2024-12-08T21:14:18,801 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e57e248fee4cbf8a4ec4c19d9510bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733692457806 2024-12-08T21:14:18,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2480f3296a364fe4a20b18e83cb5ee7e is 50, key is test_row_0/A:col10/1733692458701/Put/seqid=0 2024-12-08T21:14:18,804 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 39a5da8de49f4d7ea4bcf6ad69e12565, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733692457940 2024-12-08T21:14:18,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:18,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:18,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:18,825 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:18,826 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/310a8c4105cb4d469193e7608090a487 is 50, key is test_row_0/B:col10/1733692458582/Put/seqid=0 2024-12-08T21:14:18,836 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:18,836 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e3a1c6db921d48148d07406cc21448fc is 50, key is test_row_0/C:col10/1733692458582/Put/seqid=0 2024-12-08T21:14:18,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692518843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692518844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692518847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741894_1070 (size=12301) 2024-12-08T21:14:18,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741895_1071 (size=12949) 2024-12-08T21:14:18,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741896_1072 (size=12949) 2024-12-08T21:14:18,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692518958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692518958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692518958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:18,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692518967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:18,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:18,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:14:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692519148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,153 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692519168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692519168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692519168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,256 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2480f3296a364fe4a20b18e83cb5ee7e 2024-12-08T21:14:19,263 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/310a8c4105cb4d469193e7608090a487 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/310a8c4105cb4d469193e7608090a487 2024-12-08T21:14:19,265 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e3a1c6db921d48148d07406cc21448fc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a1c6db921d48148d07406cc21448fc 2024-12-08T21:14:19,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c6881758744f47388e762391389b4065 is 50, key is test_row_0/B:col10/1733692458701/Put/seqid=0 2024-12-08T21:14:19,277 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into 310a8c4105cb4d469193e7608090a487(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:19,277 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:19,277 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692458789; duration=0sec 2024-12-08T21:14:19,277 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:19,277 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:19,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692519277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,280 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into e3a1c6db921d48148d07406cc21448fc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:19,280 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:19,281 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692458796; duration=0sec 2024-12-08T21:14:19,281 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:19,281 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:19,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741897_1073 (size=12301) 2024-12-08T21:14:19,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c6881758744f47388e762391389b4065 2024-12-08T21:14:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/6644bae8cd5547e5bf2cabba5157efae is 50, key is test_row_0/C:col10/1733692458701/Put/seqid=0 2024-12-08T21:14:19,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741898_1074 (size=12301) 2024-12-08T21:14:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692519478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692519478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692519478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,701 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/6644bae8cd5547e5bf2cabba5157efae 2024-12-08T21:14:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/2480f3296a364fe4a20b18e83cb5ee7e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2480f3296a364fe4a20b18e83cb5ee7e 2024-12-08T21:14:19,712 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2480f3296a364fe4a20b18e83cb5ee7e, entries=150, sequenceid=411, filesize=12.0 K 2024-12-08T21:14:19,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c6881758744f47388e762391389b4065 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6881758744f47388e762391389b4065 2024-12-08T21:14:19,718 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6881758744f47388e762391389b4065, entries=150, sequenceid=411, filesize=12.0 K 2024-12-08T21:14:19,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/6644bae8cd5547e5bf2cabba5157efae as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/6644bae8cd5547e5bf2cabba5157efae 2024-12-08T21:14:19,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/6644bae8cd5547e5bf2cabba5157efae, entries=150, sequenceid=411, filesize=12.0 K 2024-12-08T21:14:19,726 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 0ef41e10da56e9b6857487fb3e409164 in 941ms, sequenceid=411, compaction requested=true 2024-12-08T21:14:19,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:19,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:19,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-08T21:14:19,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-08T21:14:19,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-08T21:14:19,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-12-08T21:14:19,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7610 sec 2024-12-08T21:14:19,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:19,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:14:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:19,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:19,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:19,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:19,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:19,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/1950f94999404fa2b51fc46500586c00 is 50, key is test_row_0/A:col10/1733692459789/Put/seqid=0 2024-12-08T21:14:19,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741899_1075 (size=12297) 2024-12-08T21:14:19,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/1950f94999404fa2b51fc46500586c00 2024-12-08T21:14:19,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/8c6cdd6b3f2a4e09a6deec9ceaad5471 is 50, key is test_row_0/B:col10/1733692459789/Put/seqid=0 2024-12-08T21:14:19,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741900_1076 (size=9857) 2024-12-08T21:14:19,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/8c6cdd6b3f2a4e09a6deec9ceaad5471 2024-12-08T21:14:19,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692519836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c7ddc80c24ec48839d91eda710f43db8 is 50, key is test_row_0/C:col10/1733692459789/Put/seqid=0 2024-12-08T21:14:19,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741901_1077 (size=9857) 2024-12-08T21:14:19,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c7ddc80c24ec48839d91eda710f43db8 2024-12-08T21:14:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/1950f94999404fa2b51fc46500586c00 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/1950f94999404fa2b51fc46500586c00 2024-12-08T21:14:19,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/1950f94999404fa2b51fc46500586c00, entries=150, sequenceid=425, filesize=12.0 K 2024-12-08T21:14:19,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/8c6cdd6b3f2a4e09a6deec9ceaad5471 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8c6cdd6b3f2a4e09a6deec9ceaad5471 2024-12-08T21:14:19,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8c6cdd6b3f2a4e09a6deec9ceaad5471, entries=100, sequenceid=425, filesize=9.6 K 2024-12-08T21:14:19,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c7ddc80c24ec48839d91eda710f43db8 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c7ddc80c24ec48839d91eda710f43db8 2024-12-08T21:14:19,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c7ddc80c24ec48839d91eda710f43db8, entries=100, sequenceid=425, filesize=9.6 K 2024-12-08T21:14:19,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0ef41e10da56e9b6857487fb3e409164 in 105ms, sequenceid=425, compaction requested=true 2024-12-08T21:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:19,895 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:19,895 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:19,896 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:19,897 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:19,897 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:19,897 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/310a8c4105cb4d469193e7608090a487, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6881758744f47388e762391389b4065, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8c6cdd6b3f2a4e09a6deec9ceaad5471] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=34.3 K 2024-12-08T21:14:19,897 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49780 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:19,897 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:19,897 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:19,897 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8367fb52fef846919bbb185a5f9a3b66, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/91b64a6482674308ae6af99b0c490b78, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2480f3296a364fe4a20b18e83cb5ee7e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/1950f94999404fa2b51fc46500586c00] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=48.6 K 2024-12-08T21:14:19,898 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 310a8c4105cb4d469193e7608090a487, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733692457940 2024-12-08T21:14:19,898 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8367fb52fef846919bbb185a5f9a3b66, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733692457806 2024-12-08T21:14:19,899 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c6881758744f47388e762391389b4065, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733692458694 2024-12-08T21:14:19,899 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b64a6482674308ae6af99b0c490b78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733692457940 2024-12-08T21:14:19,899 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6cdd6b3f2a4e09a6deec9ceaad5471, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733692458842 2024-12-08T21:14:19,900 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2480f3296a364fe4a20b18e83cb5ee7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733692458694 2024-12-08T21:14:19,901 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1950f94999404fa2b51fc46500586c00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733692458842 2024-12-08T21:14:19,910 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:19,911 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c987c0b7584243c19866601002ee5b14 is 50, key is test_row_0/B:col10/1733692459789/Put/seqid=0 2024-12-08T21:14:19,927 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:19,928 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/02b8f4e645974bc8bb34f08012fa8a91 is 50, key is test_row_0/A:col10/1733692459789/Put/seqid=0 2024-12-08T21:14:19,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741902_1078 (size=13051) 2024-12-08T21:14:19,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741903_1079 (size=13017) 2024-12-08T21:14:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:19,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T21:14:19,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:19,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:19,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:19,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:19,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:19,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:19,952 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c987c0b7584243c19866601002ee5b14 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c987c0b7584243c19866601002ee5b14 2024-12-08T21:14:19,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/8557dc69a874484c8b037bfa493a4572 is 50, key is test_row_0/A:col10/1733692459834/Put/seqid=0 2024-12-08T21:14:19,964 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/02b8f4e645974bc8bb34f08012fa8a91 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/02b8f4e645974bc8bb34f08012fa8a91 2024-12-08T21:14:19,968 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into c987c0b7584243c19866601002ee5b14(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:19,968 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:19,968 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692459895; duration=0sec 2024-12-08T21:14:19,972 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:19,972 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:19,972 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:19,973 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:19,973 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:19,973 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:19,974 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a1c6db921d48148d07406cc21448fc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/6644bae8cd5547e5bf2cabba5157efae, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c7ddc80c24ec48839d91eda710f43db8] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=34.3 K 2024-12-08T21:14:19,974 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 02b8f4e645974bc8bb34f08012fa8a91(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:19,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:19,974 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=12, startTime=1733692459895; duration=0sec 2024-12-08T21:14:19,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a1c6db921d48148d07406cc21448fc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733692457940 2024-12-08T21:14:19,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:19,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:19,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6644bae8cd5547e5bf2cabba5157efae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733692458694 2024-12-08T21:14:19,975 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ddc80c24ec48839d91eda710f43db8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733692458842 2024-12-08T21:14:19,987 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:19,987 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/750ccca067bd497f811ed5517470c54b is 50, key is test_row_0/C:col10/1733692459789/Put/seqid=0 2024-12-08T21:14:19,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741904_1080 (size=14741) 2024-12-08T21:14:19,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692519990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:19,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:19,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/8557dc69a874484c8b037bfa493a4572 2024-12-08T21:14:20,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692519996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692519997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692520002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/4b9b68ca174046c69384c1122ab897c1 is 50, key is test_row_0/B:col10/1733692459834/Put/seqid=0 2024-12-08T21:14:20,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741905_1081 (size=13051) 2024-12-08T21:14:20,037 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/750ccca067bd497f811ed5517470c54b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/750ccca067bd497f811ed5517470c54b 2024-12-08T21:14:20,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741906_1082 (size=12301) 2024-12-08T21:14:20,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/4b9b68ca174046c69384c1122ab897c1 2024-12-08T21:14:20,053 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into 750ccca067bd497f811ed5517470c54b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:20,054 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:20,054 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692459895; duration=0sec 2024-12-08T21:14:20,054 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:20,054 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:20,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/94f264571b3346f892eeb9f179870efb is 50, key is test_row_0/C:col10/1733692459834/Put/seqid=0 2024-12-08T21:14:20,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741907_1083 (size=12301) 2024-12-08T21:14:20,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/94f264571b3346f892eeb9f179870efb 2024-12-08T21:14:20,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/8557dc69a874484c8b037bfa493a4572 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8557dc69a874484c8b037bfa493a4572 2024-12-08T21:14:20,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8557dc69a874484c8b037bfa493a4572, entries=200, sequenceid=446, filesize=14.4 K 2024-12-08T21:14:20,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/4b9b68ca174046c69384c1122ab897c1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4b9b68ca174046c69384c1122ab897c1 2024-12-08T21:14:20,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692520108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692520108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692520110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4b9b68ca174046c69384c1122ab897c1, entries=150, sequenceid=446, filesize=12.0 K 2024-12-08T21:14:20,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/94f264571b3346f892eeb9f179870efb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94f264571b3346f892eeb9f179870efb 2024-12-08T21:14:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-08T21:14:20,119 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:20,119 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-08T21:14:20,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94f264571b3346f892eeb9f179870efb, entries=150, sequenceid=446, filesize=12.0 K 2024-12-08T21:14:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T21:14:20,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 0ef41e10da56e9b6857487fb3e409164 in 173ms, sequenceid=446, compaction requested=false 2024-12-08T21:14:20,125 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:20,127 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:20,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T21:14:20,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-08T21:14:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:20,280 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:14:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:20,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:20,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/236a86643591474bb83af81ec50ab9cd is 50, key is test_row_0/A:col10/1733692459984/Put/seqid=0 2024-12-08T21:14:20,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741908_1084 (size=12301) 2024-12-08T21:14:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:20,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692520374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733692460342, max timestamp=1733692460376Num uniques 859; 2024-12-08T21:14:20,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733692460342, max timestamp=1733692460376Num uniques 859; 2024-12-08T21:14:20,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:20,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733692460342, max timestamp=1733692460376Num uniques 859; 2024-12-08T21:14:20,379 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:20,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692520405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692520408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T21:14:20,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692520488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692520518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692520518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,696 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/236a86643591474bb83af81ec50ab9cd 2024-12-08T21:14:20,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692520698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/64cde1237107484a990e8a83ce795bd6 is 50, key is test_row_0/B:col10/1733692459984/Put/seqid=0 2024-12-08T21:14:20,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741909_1085 (size=12301) 2024-12-08T21:14:20,715 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/64cde1237107484a990e8a83ce795bd6 2024-12-08T21:14:20,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c5271671995c4e909034ad940b9b98c5 is 50, key is test_row_0/C:col10/1733692459984/Put/seqid=0 2024-12-08T21:14:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692520728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:20,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692520728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741910_1086 (size=12301) 2024-12-08T21:14:20,733 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c5271671995c4e909034ad940b9b98c5 2024-12-08T21:14:20,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/236a86643591474bb83af81ec50ab9cd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/236a86643591474bb83af81ec50ab9cd 2024-12-08T21:14:20,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/236a86643591474bb83af81ec50ab9cd, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T21:14:20,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/64cde1237107484a990e8a83ce795bd6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/64cde1237107484a990e8a83ce795bd6 2024-12-08T21:14:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T21:14:20,750 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/64cde1237107484a990e8a83ce795bd6, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T21:14:20,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/c5271671995c4e909034ad940b9b98c5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c5271671995c4e909034ad940b9b98c5 2024-12-08T21:14:20,758 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c5271671995c4e909034ad940b9b98c5, entries=150, sequenceid=461, filesize=12.0 K 2024-12-08T21:14:20,759 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 0ef41e10da56e9b6857487fb3e409164 in 479ms, sequenceid=461, compaction requested=true 2024-12-08T21:14:20,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:20,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:20,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-08T21:14:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-08T21:14:20,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-08T21:14:20,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 634 msec 2024-12-08T21:14:20,765 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 643 msec 2024-12-08T21:14:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:21,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-08T21:14:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:21,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:21,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:21,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5cba7ca9d36f46c19d1436cf0f2a0bba is 50, key is test_row_0/A:col10/1733692461008/Put/seqid=0 2024-12-08T21:14:21,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692521026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741911_1087 (size=14741) 2024-12-08T21:14:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5cba7ca9d36f46c19d1436cf0f2a0bba 2024-12-08T21:14:21,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692521029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692521038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/01243b48587a4192bed6b240951f2d8f is 50, key is test_row_0/B:col10/1733692461008/Put/seqid=0 2024-12-08T21:14:21,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692521038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741912_1088 (size=12301) 2024-12-08T21:14:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692521138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692521138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692521167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,172 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-08T21:14:21,258 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:21,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-08T21:14:21,261 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T21:14:21,263 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:21,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692521348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692521348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T21:14:21,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T21:14:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/01243b48587a4192bed6b240951f2d8f 2024-12-08T21:14:21,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/aff642c3104e4c8e83d352696be221c0 is 50, key is test_row_0/C:col10/1733692461008/Put/seqid=0 2024-12-08T21:14:21,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741913_1089 (size=12301) 2024-12-08T21:14:21,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692521548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692521558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T21:14:21,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T21:14:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692521657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692521658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:21,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:21,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T21:14:21,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:21,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/aff642c3104e4c8e83d352696be221c0 2024-12-08T21:14:21,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T21:14:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:21,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/5cba7ca9d36f46c19d1436cf0f2a0bba as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5cba7ca9d36f46c19d1436cf0f2a0bba 2024-12-08T21:14:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T21:14:21,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5cba7ca9d36f46c19d1436cf0f2a0bba, entries=200, sequenceid=497, filesize=14.4 K 2024-12-08T21:14:21,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/01243b48587a4192bed6b240951f2d8f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/01243b48587a4192bed6b240951f2d8f 2024-12-08T21:14:21,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/01243b48587a4192bed6b240951f2d8f, entries=150, sequenceid=497, filesize=12.0 K 2024-12-08T21:14:21,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/aff642c3104e4c8e83d352696be221c0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/aff642c3104e4c8e83d352696be221c0 2024-12-08T21:14:21,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/aff642c3104e4c8e83d352696be221c0, entries=150, sequenceid=497, filesize=12.0 K 2024-12-08T21:14:21,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 0ef41e10da56e9b6857487fb3e409164 in 902ms, sequenceid=497, compaction requested=true 2024-12-08T21:14:21,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:21,913 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:21,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:21,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:21,913 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:21,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:21,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:21,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:21,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:21,918 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54800 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:21,918 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:21,918 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,919 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/02b8f4e645974bc8bb34f08012fa8a91, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8557dc69a874484c8b037bfa493a4572, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/236a86643591474bb83af81ec50ab9cd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5cba7ca9d36f46c19d1436cf0f2a0bba] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=53.5 K 2024-12-08T21:14:21,919 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:21,919 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:21,919 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,919 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c987c0b7584243c19866601002ee5b14, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4b9b68ca174046c69384c1122ab897c1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/64cde1237107484a990e8a83ce795bd6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/01243b48587a4192bed6b240951f2d8f] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=48.8 K 2024-12-08T21:14:21,919 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02b8f4e645974bc8bb34f08012fa8a91, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733692458694 2024-12-08T21:14:21,919 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c987c0b7584243c19866601002ee5b14, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733692458694 2024-12-08T21:14:21,920 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9b68ca174046c69384c1122ab897c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733692459831 2024-12-08T21:14:21,920 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8557dc69a874484c8b037bfa493a4572, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733692459820 2024-12-08T21:14:21,920 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 236a86643591474bb83af81ec50ab9cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733692459984 2024-12-08T21:14:21,920 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 64cde1237107484a990e8a83ce795bd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733692459984 2024-12-08T21:14:21,921 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cba7ca9d36f46c19d1436cf0f2a0bba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733692460402 2024-12-08T21:14:21,921 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 01243b48587a4192bed6b240951f2d8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733692460402 2024-12-08T21:14:21,936 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:21,937 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/7691987fdca04c3ba4c501d4da18e510 is 50, key is test_row_0/A:col10/1733692461008/Put/seqid=0 2024-12-08T21:14:21,941 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#77 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:21,941 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/30552b4b361f48eeb6076096281e15a0 is 50, key is test_row_0/B:col10/1733692461008/Put/seqid=0 2024-12-08T21:14:21,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741915_1091 (size=13187) 2024-12-08T21:14:21,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741914_1090 (size=13153) 2024-12-08T21:14:21,985 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/7691987fdca04c3ba4c501d4da18e510 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7691987fdca04c3ba4c501d4da18e510 2024-12-08T21:14:21,996 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 7691987fdca04c3ba4c501d4da18e510(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:21,996 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:21,996 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=12, startTime=1733692461913; duration=0sec 2024-12-08T21:14:21,996 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:21,996 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:21,996 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:21,998 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:21,998 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:21,998 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:21,999 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/750ccca067bd497f811ed5517470c54b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94f264571b3346f892eeb9f179870efb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c5271671995c4e909034ad940b9b98c5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/aff642c3104e4c8e83d352696be221c0] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=48.8 K 2024-12-08T21:14:21,999 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750ccca067bd497f811ed5517470c54b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1733692458694 2024-12-08T21:14:22,000 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94f264571b3346f892eeb9f179870efb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733692459831 2024-12-08T21:14:22,000 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5271671995c4e909034ad940b9b98c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733692459984 2024-12-08T21:14:22,001 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting aff642c3104e4c8e83d352696be221c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733692460402 2024-12-08T21:14:22,012 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:22,012 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/02ad3f102a90418184611bff8898f38a is 50, key is test_row_0/C:col10/1733692461008/Put/seqid=0 2024-12-08T21:14:22,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741916_1092 (size=13187) 2024-12-08T21:14:22,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-08T21:14:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:22,030 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:22,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:22,033 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/02ad3f102a90418184611bff8898f38a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/02ad3f102a90418184611bff8898f38a 2024-12-08T21:14:22,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e8afd791bc184ff886a62d1092417ca4 is 50, key is test_row_0/A:col10/1733692461019/Put/seqid=0 2024-12-08T21:14:22,044 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into 02ad3f102a90418184611bff8898f38a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:22,044 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:22,045 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=12, startTime=1733692461914; duration=0sec 2024-12-08T21:14:22,045 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:22,045 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:22,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741917_1093 (size=12301) 2024-12-08T21:14:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:22,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692462184, max timestamp=1733692462210Num uniques 859; 2024-12-08T21:14:22,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692462184, max timestamp=1733692462210Num uniques 859; 2024-12-08T21:14:22,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:14:22,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:14:22,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692462184, max timestamp=1733692462210Num uniques 859; 2024-12-08T21:14:22,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:14:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692522244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692522245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692522358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692522358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,381 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/30552b4b361f48eeb6076096281e15a0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/30552b4b361f48eeb6076096281e15a0 2024-12-08T21:14:22,392 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into 30552b4b361f48eeb6076096281e15a0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:22,392 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:22,392 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=12, startTime=1733692461913; duration=0sec 2024-12-08T21:14:22,392 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:22,392 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T21:14:22,456 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e8afd791bc184ff886a62d1092417ca4 2024-12-08T21:14:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/8115a861217248639d05349c5d525d60 is 50, key is test_row_0/B:col10/1733692461019/Put/seqid=0 2024-12-08T21:14:22,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741918_1094 (size=12301) 2024-12-08T21:14:22,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692522568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692522568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692522568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,572 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692522577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,581 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,871 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/8115a861217248639d05349c5d525d60 2024-12-08T21:14:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692522886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692522886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:22,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/f4d8fb0de1f3458abfda1e9b23a9143a is 50, key is test_row_0/C:col10/1733692461019/Put/seqid=0 2024-12-08T21:14:22,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741919_1095 (size=12301) 2024-12-08T21:14:23,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/f4d8fb0de1f3458abfda1e9b23a9143a 2024-12-08T21:14:23,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e8afd791bc184ff886a62d1092417ca4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e8afd791bc184ff886a62d1092417ca4 2024-12-08T21:14:23,317 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e8afd791bc184ff886a62d1092417ca4, entries=150, sequenceid=508, filesize=12.0 K 2024-12-08T21:14:23,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/8115a861217248639d05349c5d525d60 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8115a861217248639d05349c5d525d60 2024-12-08T21:14:23,323 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8115a861217248639d05349c5d525d60, entries=150, sequenceid=508, filesize=12.0 K 2024-12-08T21:14:23,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/f4d8fb0de1f3458abfda1e9b23a9143a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/f4d8fb0de1f3458abfda1e9b23a9143a 2024-12-08T21:14:23,329 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/f4d8fb0de1f3458abfda1e9b23a9143a, entries=150, sequenceid=508, filesize=12.0 K 2024-12-08T21:14:23,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0ef41e10da56e9b6857487fb3e409164 in 1300ms, sequenceid=508, compaction requested=false 2024-12-08T21:14:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:23,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-08T21:14:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-08T21:14:23,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-08T21:14:23,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0680 sec 2024-12-08T21:14:23,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.0750 sec 2024-12-08T21:14:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:23,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-08T21:14:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:23,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:23,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:23,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/0178b50ee2dd429e8116e17e189da759 is 50, key is test_row_0/A:col10/1733692462243/Put/seqid=0 2024-12-08T21:14:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-08T21:14:23,408 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:23,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-08T21:14:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:14:23,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741920_1096 (size=17181) 2024-12-08T21:14:23,411 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:23,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/0178b50ee2dd429e8116e17e189da759 2024-12-08T21:14:23,412 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:23,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:23,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:23,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692523410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:23,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:23,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692523416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:23,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/1d78a94b76334df49311d87734e9a034 is 50, key is test_row_0/B:col10/1733692462243/Put/seqid=0 2024-12-08T21:14:23,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741921_1097 (size=12301) 2024-12-08T21:14:23,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:14:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692523518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:23,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692523528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:23,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:23,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T21:14:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:23,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T21:14:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:23,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:14:23,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692523728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:23,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692523738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:23,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:23,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T21:14:23,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:23,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:23,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:23,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:23,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/1d78a94b76334df49311d87734e9a034 2024-12-08T21:14:23,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e78b5ad8c08148679b545cbd23550e0b is 50, key is test_row_0/C:col10/1733692462243/Put/seqid=0 2024-12-08T21:14:23,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741922_1098 (size=12301) 2024-12-08T21:14:24,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T21:14:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:24,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:14:24,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692524038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692524048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T21:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e78b5ad8c08148679b545cbd23550e0b 2024-12-08T21:14:24,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/0178b50ee2dd429e8116e17e189da759 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/0178b50ee2dd429e8116e17e189da759 2024-12-08T21:14:24,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/0178b50ee2dd429e8116e17e189da759, entries=250, sequenceid=547, filesize=16.8 K 2024-12-08T21:14:24,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/1d78a94b76334df49311d87734e9a034 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/1d78a94b76334df49311d87734e9a034 2024-12-08T21:14:24,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T21:14:24,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:24,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:24,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/1d78a94b76334df49311d87734e9a034, entries=150, sequenceid=547, filesize=12.0 K 2024-12-08T21:14:24,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e78b5ad8c08148679b545cbd23550e0b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e78b5ad8c08148679b545cbd23550e0b 2024-12-08T21:14:24,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e78b5ad8c08148679b545cbd23550e0b, entries=150, sequenceid=547, filesize=12.0 K 2024-12-08T21:14:24,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 0ef41e10da56e9b6857487fb3e409164 in 976ms, sequenceid=547, compaction requested=true 2024-12-08T21:14:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:24,377 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:24,377 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:24,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:24,383 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:24,383 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:24,383 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:24,383 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:24,383 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,383 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,384 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7691987fdca04c3ba4c501d4da18e510, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e8afd791bc184ff886a62d1092417ca4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/0178b50ee2dd429e8116e17e189da759] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=41.6 K 2024-12-08T21:14:24,384 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/30552b4b361f48eeb6076096281e15a0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8115a861217248639d05349c5d525d60, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/1d78a94b76334df49311d87734e9a034] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=36.9 K 2024-12-08T21:14:24,386 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 30552b4b361f48eeb6076096281e15a0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733692460402 2024-12-08T21:14:24,386 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7691987fdca04c3ba4c501d4da18e510, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733692460402 2024-12-08T21:14:24,387 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8115a861217248639d05349c5d525d60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733692461019 2024-12-08T21:14:24,388 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d78a94b76334df49311d87734e9a034, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733692462243 2024-12-08T21:14:24,388 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8afd791bc184ff886a62d1092417ca4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733692461019 2024-12-08T21:14:24,390 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0178b50ee2dd429e8116e17e189da759, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733692462222 2024-12-08T21:14:24,409 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:24,411 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/7b0e9ba8e2654a6fb0cd8585aa3cbaa9 is 50, key is test_row_0/A:col10/1733692462243/Put/seqid=0 2024-12-08T21:14:24,419 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:24,420 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/dea6fb29131c430da5b203aa9e366832 is 50, key is test_row_0/B:col10/1733692462243/Put/seqid=0 2024-12-08T21:14:24,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741923_1099 (size=13255) 2024-12-08T21:14:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741924_1100 (size=13289) 2024-12-08T21:14:24,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-08T21:14:24,490 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/dea6fb29131c430da5b203aa9e366832 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/dea6fb29131c430da5b203aa9e366832 2024-12-08T21:14:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,490 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T21:14:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:24,502 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into dea6fb29131c430da5b203aa9e366832(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:24,502 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:24,502 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692464377; duration=0sec 2024-12-08T21:14:24,503 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:24,503 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:24,503 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:24,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e3f2531b60d8431689d613482b7dfb7f is 50, key is test_row_0/A:col10/1733692463413/Put/seqid=0 2024-12-08T21:14:24,504 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:24,504 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:24,504 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:24,505 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/02ad3f102a90418184611bff8898f38a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/f4d8fb0de1f3458abfda1e9b23a9143a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e78b5ad8c08148679b545cbd23550e0b] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=36.9 K 2024-12-08T21:14:24,506 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ad3f102a90418184611bff8898f38a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=497, earliestPutTs=1733692460402 2024-12-08T21:14:24,507 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d8fb0de1f3458abfda1e9b23a9143a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1733692461019 2024-12-08T21:14:24,507 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e78b5ad8c08148679b545cbd23550e0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733692462243 2024-12-08T21:14:24,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741925_1101 (size=12301) 2024-12-08T21:14:24,532 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e3f2531b60d8431689d613482b7dfb7f 2024-12-08T21:14:24,537 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#89 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:24,538 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/b3319e886cb44f37ae22c3e9fd791779 is 50, key is test_row_0/C:col10/1733692462243/Put/seqid=0 2024-12-08T21:14:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:14:24,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9827ab06318d449ea16f214fb52b0a08 is 50, key is test_row_0/B:col10/1733692463413/Put/seqid=0 2024-12-08T21:14:24,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:24,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:24,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741926_1102 (size=13289) 2024-12-08T21:14:24,578 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/b3319e886cb44f37ae22c3e9fd791779 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/b3319e886cb44f37ae22c3e9fd791779 2024-12-08T21:14:24,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741927_1103 (size=12301) 2024-12-08T21:14:24,586 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into b3319e886cb44f37ae22c3e9fd791779(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:24,586 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:24,586 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692464377; duration=0sec 2024-12-08T21:14:24,587 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:24,587 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:24,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692464586, max timestamp=1733692464602Num uniques 859; 2024-12-08T21:14:24,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:24,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692464586, max timestamp=1733692464602Num uniques 859; 2024-12-08T21:14:24,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692464586, max timestamp=1733692464602Num uniques 859; 2024-12-08T21:14:24,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:24,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:24,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733692464615, max timestamp=1733692464630Num uniques 1068; 2024-12-08T21:14:24,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733692464615, max timestamp=1733692464630Num uniques 1068; 2024-12-08T21:14:24,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-08T21:14:24,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-08T21:14:24,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733692464615, max timestamp=1733692464630Num uniques 1068; 2024-12-08T21:14:24,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-08T21:14:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692524666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692524667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733692464656, max timestamp=1733692464666Num uniques 1127; 2024-12-08T21:14:24,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-12-08T21:14:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692464664, max timestamp=1733692464666Num uniques 1127; 2024-12-08T21:14:24,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692464664, max timestamp=1733692464666Num uniques 1127; 2024-12-08T21:14:24,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-08T21:14:24,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-08T21:14:24,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692524718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692464700, max timestamp=1733692464722Num uniques 1036; 2024-12-08T21:14:24,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692464700, max timestamp=1733692464722Num uniques 1136; 2024-12-08T21:14:24,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-08T21:14:24,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-12-08T21:14:24,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692464700, max timestamp=1733692464722Num uniques 1036; 2024-12-08T21:14:24,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-08T21:14:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692464732, max timestamp=1733692464784Num uniques 1095; 2024-12-08T21:14:24,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692464732, max timestamp=1733692464784Num uniques 1095; 2024-12-08T21:14:24,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-08T21:14:24,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-08T21:14:24,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692464732, max timestamp=1733692464784Num uniques 1095; 2024-12-08T21:14:24,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-08T21:14:24,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692524817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692524818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692524823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692524828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,870 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/7b0e9ba8e2654a6fb0cd8585aa3cbaa9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7b0e9ba8e2654a6fb0cd8585aa3cbaa9 2024-12-08T21:14:24,877 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 7b0e9ba8e2654a6fb0cd8585aa3cbaa9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:24,877 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:24,877 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=13, startTime=1733692464377; duration=0sec 2024-12-08T21:14:24,877 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:24,877 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:24,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692524938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692524938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692524939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:24,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9827ab06318d449ea16f214fb52b0a08 2024-12-08T21:14:24,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8afb60ca19d64109844805e6de5154e6 is 50, key is test_row_0/C:col10/1733692463413/Put/seqid=0 2024-12-08T21:14:24,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741928_1104 (size=12301) 2024-12-08T21:14:25,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692525038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692525148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692525148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692525148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47276 deadline: 1733692525198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,200 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692525348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,397 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8afb60ca19d64109844805e6de5154e6 2024-12-08T21:14:25,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/e3f2531b60d8431689d613482b7dfb7f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e3f2531b60d8431689d613482b7dfb7f 2024-12-08T21:14:25,408 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e3f2531b60d8431689d613482b7dfb7f, entries=150, sequenceid=555, filesize=12.0 K 2024-12-08T21:14:25,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9827ab06318d449ea16f214fb52b0a08 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9827ab06318d449ea16f214fb52b0a08 2024-12-08T21:14:25,416 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9827ab06318d449ea16f214fb52b0a08, entries=150, sequenceid=555, filesize=12.0 K 2024-12-08T21:14:25,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8afb60ca19d64109844805e6de5154e6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8afb60ca19d64109844805e6de5154e6 2024-12-08T21:14:25,424 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8afb60ca19d64109844805e6de5154e6, entries=150, sequenceid=555, filesize=12.0 K 2024-12-08T21:14:25,425 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 0ef41e10da56e9b6857487fb3e409164 in 935ms, sequenceid=555, compaction requested=false 2024-12-08T21:14:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-08T21:14:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-08T21:14:25,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-08T21:14:25,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0140 sec 2024-12-08T21:14:25,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.0190 sec 2024-12-08T21:14:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:25,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-12-08T21:14:25,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733692464812, max timestamp=1733692465458Num uniques 1104; 2024-12-08T21:14:25,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T21:14:25,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:25,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733692464812, max timestamp=1733692465458Num uniques 1104; 2024-12-08T21:14:25,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T21:14:25,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733692464812, max timestamp=1733692465458Num uniques 1104; 2024-12-08T21:14:25,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T21:14:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:25,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:25,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/b0f55e69118542339ea5c66444d08eec is 50, key is test_row_0/A:col10/1733692465459/Put/seqid=0 2024-12-08T21:14:25,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741929_1105 (size=17181) 2024-12-08T21:14:25,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692465481, max timestamp=1733692465490Num uniques 859; 2024-12-08T21:14:25,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692465481, max timestamp=1733692465490Num uniques 859; 2024-12-08T21:14:25,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692465481, max timestamp=1733692465490Num uniques 859; 2024-12-08T21:14:25,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:14:25,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:14:25,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:14:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692525526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692525526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692525527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-08T21:14:25,558 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:25,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-08T21:14:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T21:14:25,561 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:25,562 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:25,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:25,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692525637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692525637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692525638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T21:14:25,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T21:14:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:25,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:25,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692525847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692525847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692525847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692525858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:25,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T21:14:25,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:25,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:25,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:25,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:25,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:25,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/b0f55e69118542339ea5c66444d08eec 2024-12-08T21:14:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T21:14:25,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c4aae59ea02e40129b1919de4524d263 is 50, key is test_row_0/B:col10/1733692465459/Put/seqid=0 2024-12-08T21:14:25,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741930_1106 (size=12301) 2024-12-08T21:14:25,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c4aae59ea02e40129b1919de4524d263 2024-12-08T21:14:25,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/5f9119d873b94987a549eb9027471721 is 50, key is test_row_0/C:col10/1733692465459/Put/seqid=0 2024-12-08T21:14:25,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741931_1107 (size=12301) 2024-12-08T21:14:25,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/5f9119d873b94987a549eb9027471721 2024-12-08T21:14:25,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/b0f55e69118542339ea5c66444d08eec as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b0f55e69118542339ea5c66444d08eec 2024-12-08T21:14:25,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b0f55e69118542339ea5c66444d08eec, entries=250, sequenceid=666, filesize=16.8 K 2024-12-08T21:14:25,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/c4aae59ea02e40129b1919de4524d263 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c4aae59ea02e40129b1919de4524d263 2024-12-08T21:14:25,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c4aae59ea02e40129b1919de4524d263, entries=150, sequenceid=666, filesize=12.0 K 2024-12-08T21:14:25,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/5f9119d873b94987a549eb9027471721 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/5f9119d873b94987a549eb9027471721 2024-12-08T21:14:25,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/5f9119d873b94987a549eb9027471721, entries=150, sequenceid=666, filesize=12.0 K 2024-12-08T21:14:25,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=154.31 KB/158010 for 0ef41e10da56e9b6857487fb3e409164 in 512ms, sequenceid=666, compaction requested=true 2024-12-08T21:14:25,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:25,972 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:25,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:25,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:25,972 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:25,973 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:25,973 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:25,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:25,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:25,974 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:25,974 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:25,974 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7b0e9ba8e2654a6fb0cd8585aa3cbaa9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e3f2531b60d8431689d613482b7dfb7f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b0f55e69118542339ea5c66444d08eec] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=41.7 K 2024-12-08T21:14:25,974 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/dea6fb29131c430da5b203aa9e366832, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9827ab06318d449ea16f214fb52b0a08, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c4aae59ea02e40129b1919de4524d263] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=37.0 K 2024-12-08T21:14:25,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b0e9ba8e2654a6fb0cd8585aa3cbaa9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733692462243 2024-12-08T21:14:25,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea6fb29131c430da5b203aa9e366832, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733692462243 2024-12-08T21:14:25,975 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f2531b60d8431689d613482b7dfb7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733692463404 2024-12-08T21:14:25,975 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9827ab06318d449ea16f214fb52b0a08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733692463404 2024-12-08T21:14:25,975 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b0f55e69118542339ea5c66444d08eec, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733692464812 2024-12-08T21:14:25,975 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4aae59ea02e40129b1919de4524d263, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733692464812 2024-12-08T21:14:25,984 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:25,984 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9723b26e383644beb927eb76af8e43a6 is 50, key is test_row_0/B:col10/1733692465459/Put/seqid=0 2024-12-08T21:14:25,986 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:25,987 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/367c2102bda8422f8993dcffb038d721 is 50, key is test_row_0/A:col10/1733692465459/Put/seqid=0 2024-12-08T21:14:25,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741932_1108 (size=13391) 2024-12-08T21:14:26,004 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/9723b26e383644beb927eb76af8e43a6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9723b26e383644beb927eb76af8e43a6 2024-12-08T21:14:26,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741933_1109 (size=13357) 2024-12-08T21:14:26,015 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into 9723b26e383644beb927eb76af8e43a6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:26,016 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:26,016 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692465972; duration=0sec 2024-12-08T21:14:26,016 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:26,016 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:26,016 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:26,018 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:26,018 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:26,018 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:26,018 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/b3319e886cb44f37ae22c3e9fd791779, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8afb60ca19d64109844805e6de5154e6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/5f9119d873b94987a549eb9027471721] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=37.0 K 2024-12-08T21:14:26,019 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3319e886cb44f37ae22c3e9fd791779, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=547, earliestPutTs=1733692462243 2024-12-08T21:14:26,019 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8afb60ca19d64109844805e6de5154e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1733692463404 2024-12-08T21:14:26,020 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f9119d873b94987a549eb9027471721, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733692464812 2024-12-08T21:14:26,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-08T21:14:26,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:26,021 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-08T21:14:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:26,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/193479969ede44d18158359e312a0d02 is 50, key is test_row_0/A:col10/1733692465526/Put/seqid=0 2024-12-08T21:14:26,049 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#98 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:26,050 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/16ec566f771e458da3120fb8718e5df0 is 50, key is test_row_0/C:col10/1733692465459/Put/seqid=0 2024-12-08T21:14:26,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741934_1110 (size=12301) 2024-12-08T21:14:26,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741935_1111 (size=13391) 2024-12-08T21:14:26,077 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/16ec566f771e458da3120fb8718e5df0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/16ec566f771e458da3120fb8718e5df0 2024-12-08T21:14:26,082 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into 16ec566f771e458da3120fb8718e5df0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:26,082 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:26,082 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692465973; duration=0sec 2024-12-08T21:14:26,082 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:26,082 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:26,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692526162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692526163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692526165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T21:14:26,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692526267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692526267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692526278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,416 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/367c2102bda8422f8993dcffb038d721 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/367c2102bda8422f8993dcffb038d721 2024-12-08T21:14:26,420 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 367c2102bda8422f8993dcffb038d721(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:26,420 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:26,420 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=13, startTime=1733692465972; duration=0sec 2024-12-08T21:14:26,421 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:26,421 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:26,463 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/193479969ede44d18158359e312a0d02 2024-12-08T21:14:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/16fce10029e142639c9131c80ba7423d is 50, key is test_row_0/B:col10/1733692465526/Put/seqid=0 2024-12-08T21:14:26,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741936_1112 (size=12301) 2024-12-08T21:14:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692526478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692526478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692526487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T21:14:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692526788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692526788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692526798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692526868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:26,871 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:26,876 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/16fce10029e142639c9131c80ba7423d 2024-12-08T21:14:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8eac0e3b256e4fa783c206c33c06b9e5 is 50, key is test_row_0/C:col10/1733692465526/Put/seqid=0 2024-12-08T21:14:26,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741937_1113 (size=12301) 2024-12-08T21:14:27,290 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=704 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8eac0e3b256e4fa783c206c33c06b9e5 2024-12-08T21:14:27,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/193479969ede44d18158359e312a0d02 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/193479969ede44d18158359e312a0d02 2024-12-08T21:14:27,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692527298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692527298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:27,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:27,301 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/193479969ede44d18158359e312a0d02, entries=150, sequenceid=704, filesize=12.0 K 2024-12-08T21:14:27,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/16fce10029e142639c9131c80ba7423d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/16fce10029e142639c9131c80ba7423d 2024-12-08T21:14:27,308 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/16fce10029e142639c9131c80ba7423d, entries=150, sequenceid=704, filesize=12.0 K 2024-12-08T21:14:27,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/8eac0e3b256e4fa783c206c33c06b9e5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8eac0e3b256e4fa783c206c33c06b9e5 2024-12-08T21:14:27,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692527307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:27,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:27,316 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8eac0e3b256e4fa783c206c33c06b9e5, entries=150, sequenceid=704, filesize=12.0 K 2024-12-08T21:14:27,317 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 0ef41e10da56e9b6857487fb3e409164 in 1296ms, sequenceid=704, compaction requested=false 2024-12-08T21:14:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-08T21:14:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-08T21:14:27,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-08T21:14:27,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7560 sec 2024-12-08T21:14:27,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7630 sec 2024-12-08T21:14:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-08T21:14:27,709 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:27,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-08T21:14:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T21:14:27,712 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:27,712 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:27,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T21:14:27,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-08T21:14:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:27,865 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T21:14:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:27,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/6d6930db1f4849f3bd5d25ddb335d39e is 50, key is test_row_0/A:col10/1733692466161/Put/seqid=0 2024-12-08T21:14:27,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741938_1114 (size=9857) 2024-12-08T21:14:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T21:14:28,283 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/6d6930db1f4849f3bd5d25ddb335d39e 2024-12-08T21:14:28,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/4225a1686a734ea69bce7bc3ae83c86b is 50, key is test_row_0/B:col10/1733692466161/Put/seqid=0 2024-12-08T21:14:28,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741939_1115 (size=9857) 2024-12-08T21:14:28,300 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/4225a1686a734ea69bce7bc3ae83c86b 2024-12-08T21:14:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/9d66d4adea494179a5c2e9308f911453 is 50, key is test_row_0/C:col10/1733692466161/Put/seqid=0 2024-12-08T21:14:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:28,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:28,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741940_1116 (size=9857) 2024-12-08T21:14:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T21:14:28,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692468339, max timestamp=1733692468351Num uniques 859; 2024-12-08T21:14:28,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692468339, max timestamp=1733692468351Num uniques 859; 2024-12-08T21:14:28,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692468339, max timestamp=1733692468351Num uniques 859; 2024-12-08T21:14:28,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:28,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:28,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:28,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692528379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692528380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692468376, max timestamp=1733692468380Num uniques 1068; 2024-12-08T21:14:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T21:14:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692468376, max timestamp=1733692468380Num uniques 1068; 2024-12-08T21:14:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692468376, max timestamp=1733692468380Num uniques 1068; 2024-12-08T21:14:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T21:14:28,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-08T21:14:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733692468429, max timestamp=1733692468450Num uniques 1027; 2024-12-08T21:14:28,452 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T21:14:28,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733692468429, max timestamp=1733692468450Num uniques 1027; 2024-12-08T21:14:28,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T21:14:28,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733692468429, max timestamp=1733692468450Num uniques 1027; 2024-12-08T21:14:28,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-08T21:14:28,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692528502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=436, cellSize=19966, totalHeapSize=45584, min timestamp=1733692468491, max timestamp=1733692468507Num uniques 1086; 2024-12-08T21:14:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=436, cellSize=19966, totalHeapSize=45584, min timestamp=1733692468491, max timestamp=1733692468507Num uniques 1086; 2024-12-08T21:14:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=19966, suffix heap size=140912,new segment heap size=45584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=436 2024-12-08T21:14:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=19966, suffix heap size=140912,new segment heap size=45584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=436 2024-12-08T21:14:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692468496, max timestamp=1733692468507Num uniques 1086; 2024-12-08T21:14:28,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-08T21:14:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692528561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692528563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692528617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692528668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692528668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,724 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/9d66d4adea494179a5c2e9308f911453 2024-12-08T21:14:28,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/6d6930db1f4849f3bd5d25ddb335d39e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/6d6930db1f4849f3bd5d25ddb335d39e 2024-12-08T21:14:28,739 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/6d6930db1f4849f3bd5d25ddb335d39e, entries=100, sequenceid=714, filesize=9.6 K 2024-12-08T21:14:28,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/4225a1686a734ea69bce7bc3ae83c86b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4225a1686a734ea69bce7bc3ae83c86b 2024-12-08T21:14:28,746 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4225a1686a734ea69bce7bc3ae83c86b, entries=100, sequenceid=714, filesize=9.6 K 2024-12-08T21:14:28,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/9d66d4adea494179a5c2e9308f911453 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/9d66d4adea494179a5c2e9308f911453 2024-12-08T21:14:28,753 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/9d66d4adea494179a5c2e9308f911453, entries=100, sequenceid=714, filesize=9.6 K 2024-12-08T21:14:28,754 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=152.07 KB/155720 for 0ef41e10da56e9b6857487fb3e409164 in 889ms, sequenceid=714, compaction requested=true 2024-12-08T21:14:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-08T21:14:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-08T21:14:28,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-08T21:14:28,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0460 sec 2024-12-08T21:14:28,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.0510 sec 2024-12-08T21:14:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:28,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=158.78 KB heapSize=438.31 KB 2024-12-08T21:14:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:28,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:28,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/4775e651c9d84264ae654aa67777d8ed is 50, key is test_row_0/A:col10/1733692468560/Put/seqid=0 2024-12-08T21:14:28,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692528848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-08T21:14:28,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:28,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741941_1117 (size=14741) 2024-12-08T21:14:28,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-08T21:14:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:14:28,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:28,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:28,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:28,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692528878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692528878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47262 deadline: 1733692528898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,902 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692528958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:14:29,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692529168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:29,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:14:29,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692529188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692529188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:29,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/4775e651c9d84264ae654aa67777d8ed 2024-12-08T21:14:29,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/97ac242636d246ca8b0bfde76f10f652 is 50, key is test_row_0/B:col10/1733692468560/Put/seqid=0 2024-12-08T21:14:29,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741942_1118 (size=12301) 2024-12-08T21:14:29,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692529478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:14:29,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/97ac242636d246ca8b0bfde76f10f652 2024-12-08T21:14:29,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/54798e2505fa40e38856c734caf9af2b is 50, key is test_row_0/C:col10/1733692468560/Put/seqid=0 2024-12-08T21:14:29,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741943_1119 (size=12301) 2024-12-08T21:14:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692529698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692529698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:29,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:29,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:29,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:29,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:29,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692529988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:14:30,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:30,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:30,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:30,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=802 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/54798e2505fa40e38856c734caf9af2b 2024-12-08T21:14:30,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/4775e651c9d84264ae654aa67777d8ed as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/4775e651c9d84264ae654aa67777d8ed 2024-12-08T21:14:30,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/4775e651c9d84264ae654aa67777d8ed, entries=200, sequenceid=802, filesize=14.4 K 2024-12-08T21:14:30,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/97ac242636d246ca8b0bfde76f10f652 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/97ac242636d246ca8b0bfde76f10f652 2024-12-08T21:14:30,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/97ac242636d246ca8b0bfde76f10f652, entries=150, sequenceid=802, filesize=12.0 K 2024-12-08T21:14:30,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/54798e2505fa40e38856c734caf9af2b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/54798e2505fa40e38856c734caf9af2b 2024-12-08T21:14:30,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/54798e2505fa40e38856c734caf9af2b, entries=150, sequenceid=802, filesize=12.0 K 2024-12-08T21:14:30,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~158.78 KB/162590, heapSize ~438.27 KB/448784, currentSize=26.84 KB/27480 for 0ef41e10da56e9b6857487fb3e409164 in 1284ms, sequenceid=802, compaction requested=true 2024-12-08T21:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:30,118 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:30,118 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:30,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:30,119 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50256 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:30,119 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:30,119 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:30,119 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:30,119 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:30,119 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:30,120 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/367c2102bda8422f8993dcffb038d721, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/193479969ede44d18158359e312a0d02, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/6d6930db1f4849f3bd5d25ddb335d39e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/4775e651c9d84264ae654aa67777d8ed] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=49.1 K 2024-12-08T21:14:30,120 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9723b26e383644beb927eb76af8e43a6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/16fce10029e142639c9131c80ba7423d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4225a1686a734ea69bce7bc3ae83c86b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/97ac242636d246ca8b0bfde76f10f652] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=46.7 K 2024-12-08T21:14:30,120 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 367c2102bda8422f8993dcffb038d721, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733692464812 2024-12-08T21:14:30,120 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 9723b26e383644beb927eb76af8e43a6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733692464812 2024-12-08T21:14:30,120 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 193479969ede44d18158359e312a0d02, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733692465509 2024-12-08T21:14:30,120 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 16fce10029e142639c9131c80ba7423d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733692465509 2024-12-08T21:14:30,121 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d6930db1f4849f3bd5d25ddb335d39e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733692466161 2024-12-08T21:14:30,121 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4225a1686a734ea69bce7bc3ae83c86b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733692466161 2024-12-08T21:14:30,121 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4775e651c9d84264ae654aa67777d8ed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1733692468551 2024-12-08T21:14:30,122 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 97ac242636d246ca8b0bfde76f10f652, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1733692468559 2024-12-08T21:14:30,129 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:30,129 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:30,130 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/242b9f2c53ac44a8a8ab80b0655260af is 50, key is test_row_0/A:col10/1733692468560/Put/seqid=0 2024-12-08T21:14:30,130 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/e83b234a7fff44ce8abf39c467a5ff5c is 50, key is test_row_0/B:col10/1733692468560/Put/seqid=0 2024-12-08T21:14:30,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741944_1120 (size=13493) 2024-12-08T21:14:30,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741945_1121 (size=13527) 2024-12-08T21:14:30,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-08T21:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:30,243 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T21:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:30,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:30,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/57d33786b1a64e099d78a6ea68cecea1 is 50, key is test_row_1/A:col10/1733692468846/Put/seqid=0 2024-12-08T21:14:30,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741946_1122 (size=9857) 2024-12-08T21:14:30,541 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/e83b234a7fff44ce8abf39c467a5ff5c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e83b234a7fff44ce8abf39c467a5ff5c 2024-12-08T21:14:30,541 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/242b9f2c53ac44a8a8ab80b0655260af as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/242b9f2c53ac44a8a8ab80b0655260af 2024-12-08T21:14:30,547 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into e83b234a7fff44ce8abf39c467a5ff5c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:30,547 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:30,547 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=12, startTime=1733692470118; duration=0sec 2024-12-08T21:14:30,548 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into 242b9f2c53ac44a8a8ab80b0655260af(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:30,548 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:30,548 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=12, startTime=1733692470118; duration=0sec 2024-12-08T21:14:30,548 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:30,548 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:30,548 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:30,548 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:30,548 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:30,549 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:30,550 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:30,550 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:30,550 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/16ec566f771e458da3120fb8718e5df0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8eac0e3b256e4fa783c206c33c06b9e5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/9d66d4adea494179a5c2e9308f911453, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/54798e2505fa40e38856c734caf9af2b] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=46.7 K 2024-12-08T21:14:30,551 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ec566f771e458da3120fb8718e5df0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733692464812 2024-12-08T21:14:30,551 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eac0e3b256e4fa783c206c33c06b9e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=704, earliestPutTs=1733692465509 2024-12-08T21:14:30,552 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d66d4adea494179a5c2e9308f911453, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1733692466161 2024-12-08T21:14:30,552 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 54798e2505fa40e38856c734caf9af2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1733692468559 2024-12-08T21:14:30,563 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#110 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:30,563 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/7bb201998ece4a6f89463b8eac41b037 is 50, key is test_row_0/C:col10/1733692468560/Put/seqid=0 2024-12-08T21:14:30,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741947_1123 (size=13527) 2024-12-08T21:14:30,580 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/7bb201998ece4a6f89463b8eac41b037 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/7bb201998ece4a6f89463b8eac41b037 2024-12-08T21:14:30,588 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into 7bb201998ece4a6f89463b8eac41b037(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:30,588 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:30,588 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=12, startTime=1733692470118; duration=0sec 2024-12-08T21:14:30,588 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:30,588 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:30,653 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=809 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/57d33786b1a64e099d78a6ea68cecea1 2024-12-08T21:14:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/21099320c5b14a49b799ed463f0c4a1d is 50, key is test_row_1/B:col10/1733692468846/Put/seqid=0 2024-12-08T21:14:30,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741948_1124 (size=9857) 2024-12-08T21:14:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:30,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. as already flushing 2024-12-08T21:14:30,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692470726, max timestamp=1733692470734Num uniques 859; 2024-12-08T21:14:30,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:14:30,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692470726, max timestamp=1733692470734Num uniques 859; 2024-12-08T21:14:30,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692470726, max timestamp=1733692470734Num uniques 859; 2024-12-08T21:14:30,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:14:30,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:14:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692530757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:30,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733692470750, max timestamp=1733692470756Num uniques 1168; 2024-12-08T21:14:30,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-12-08T21:14:30,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733692470750, max timestamp=1733692470756Num uniques 1168; 2024-12-08T21:14:30,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-12-08T21:14:30,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733692470750, max timestamp=1733692470756Num uniques 1168; 2024-12-08T21:14:30,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-12-08T21:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:30,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692470794, max timestamp=1733692470810Num uniques 1127; 2024-12-08T21:14:30,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-08T21:14:30,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692470794, max timestamp=1733692470810Num uniques 1127; 2024-12-08T21:14:30,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-08T21:14:30,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692470794, max timestamp=1733692470810Num uniques 1127; 2024-12-08T21:14:30,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-08T21:14:30,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692470832, max timestamp=1733692470854Num uniques 1036; 2024-12-08T21:14:30,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692470832, max timestamp=1733692470854Num uniques 1036; 2024-12-08T21:14:30,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T21:14:30,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T21:14:30,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692470832, max timestamp=1733692470854Num uniques 1036; 2024-12-08T21:14:30,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T21:14:30,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733692470874, max timestamp=1733692470883Num uniques 1045; 2024-12-08T21:14:30,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-08T21:14:30,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733692470874, max timestamp=1733692470883Num uniques 1045; 2024-12-08T21:14:30,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-08T21:14:30,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733692470874, max timestamp=1733692470883Num uniques 1045; 2024-12-08T21:14:30,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-08T21:14:30,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692530914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:30,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692530917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:30,994 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-08T21:14:30,994 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:30,996 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-08T21:14:30,996 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:30,996 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-08T21:14:30,996 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:30,997 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-08T21:14:30,997 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47290 deadline: 1733692530997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:30,998 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 0ef41e10da56e9b6857487fb3e409164 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:30,999 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:30,999 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:30,999 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:30,999 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,000 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,000 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:31,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:31,000 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:31,000 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:14:31,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:31,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692531028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692531028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:31,075 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=809 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/21099320c5b14a49b799ed463f0c4a1d 2024-12-08T21:14:31,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a757576ef9b0465a8e4a7141a510d4da is 50, key is test_row_1/C:col10/1733692468846/Put/seqid=0 2024-12-08T21:14:31,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741949_1125 (size=9857) 2024-12-08T21:14:31,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:31,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47292 deadline: 1733692531239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47304 deadline: 1733692531239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:14:31,493 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=809 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a757576ef9b0465a8e4a7141a510d4da 2024-12-08T21:14:31,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/57d33786b1a64e099d78a6ea68cecea1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/57d33786b1a64e099d78a6ea68cecea1 2024-12-08T21:14:31,508 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/57d33786b1a64e099d78a6ea68cecea1, entries=100, sequenceid=809, filesize=9.6 K 2024-12-08T21:14:31,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/21099320c5b14a49b799ed463f0c4a1d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/21099320c5b14a49b799ed463f0c4a1d 2024-12-08T21:14:31,514 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/21099320c5b14a49b799ed463f0c4a1d, entries=100, sequenceid=809, filesize=9.6 K 2024-12-08T21:14:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/a757576ef9b0465a8e4a7141a510d4da as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a757576ef9b0465a8e4a7141a510d4da 2024-12-08T21:14:31,519 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a757576ef9b0465a8e4a7141a510d4da, entries=100, sequenceid=809, filesize=9.6 K 2024-12-08T21:14:31,520 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for 0ef41e10da56e9b6857487fb3e409164 in 1276ms, sequenceid=809, compaction requested=false 2024-12-08T21:14:31,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:31,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:31,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-08T21:14:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-08T21:14:31,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-08T21:14:31,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6560 sec 2024-12-08T21:14:31,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.6640 sec 2024-12-08T21:14:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:31,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=167.72 KB heapSize=472.55 KB 2024-12-08T21:14:31,550 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-08T21:14:31,550 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-08T21:14:31,550 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:31,550 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,550 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:31,550 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:31,550 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:14:31,550 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:31,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:14:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:31,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:14:31,550 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:31,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/a56beafa25bc4875b94bfe17ff9bcbc0 is 50, key is test_row_0/A:col10/1733692470916/Put/seqid=0 2024-12-08T21:14:31,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741950_1126 (size=12301) 2024-12-08T21:14:31,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=920 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/a56beafa25bc4875b94bfe17ff9bcbc0 2024-12-08T21:14:31,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/88a31875bbdd48118f794ff55b12d51b is 50, key is test_row_0/B:col10/1733692470916/Put/seqid=0 2024-12-08T21:14:31,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741951_1127 (size=12301) 2024-12-08T21:14:32,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=920 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/88a31875bbdd48118f794ff55b12d51b 2024-12-08T21:14:32,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e9225c3d6ac340759d97f28db1e6faef is 50, key is test_row_0/C:col10/1733692470916/Put/seqid=0 2024-12-08T21:14:32,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741952_1128 (size=12301) 2024-12-08T21:14:32,442 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:14:32,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=920 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e9225c3d6ac340759d97f28db1e6faef 2024-12-08T21:14:32,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/a56beafa25bc4875b94bfe17ff9bcbc0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/a56beafa25bc4875b94bfe17ff9bcbc0 2024-12-08T21:14:32,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/a56beafa25bc4875b94bfe17ff9bcbc0, entries=150, sequenceid=920, filesize=12.0 K 2024-12-08T21:14:32,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/88a31875bbdd48118f794ff55b12d51b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/88a31875bbdd48118f794ff55b12d51b 2024-12-08T21:14:32,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/88a31875bbdd48118f794ff55b12d51b, entries=150, sequenceid=920, filesize=12.0 K 2024-12-08T21:14:32,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/e9225c3d6ac340759d97f28db1e6faef as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e9225c3d6ac340759d97f28db1e6faef 2024-12-08T21:14:32,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e9225c3d6ac340759d97f28db1e6faef, entries=150, sequenceid=920, filesize=12.0 K 2024-12-08T21:14:32,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~472.50 KB/483840, currentSize=0 B/0 for 0ef41e10da56e9b6857487fb3e409164 in 1271ms, sequenceid=920, compaction requested=true 2024-12-08T21:14:32,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:32,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:32,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:32,822 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:32,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:32,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:32,822 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:32,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0ef41e10da56e9b6857487fb3e409164:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:32,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:32,823 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:32,823 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:32,823 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/B is initiating minor compaction (all files) 2024-12-08T21:14:32,823 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/A is initiating minor compaction (all files) 2024-12-08T21:14:32,823 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/B in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:32,823 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/A in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:32,823 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e83b234a7fff44ce8abf39c467a5ff5c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/21099320c5b14a49b799ed463f0c4a1d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/88a31875bbdd48118f794ff55b12d51b] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=34.8 K 2024-12-08T21:14:32,823 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/242b9f2c53ac44a8a8ab80b0655260af, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/57d33786b1a64e099d78a6ea68cecea1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/a56beafa25bc4875b94bfe17ff9bcbc0] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=34.8 K 2024-12-08T21:14:32,823 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e83b234a7fff44ce8abf39c467a5ff5c, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1733692468559 2024-12-08T21:14:32,823 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 242b9f2c53ac44a8a8ab80b0655260af, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1733692468559 2024-12-08T21:14:32,824 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57d33786b1a64e099d78a6ea68cecea1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=809, earliestPutTs=1733692468842 2024-12-08T21:14:32,824 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 21099320c5b14a49b799ed463f0c4a1d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=809, earliestPutTs=1733692468842 2024-12-08T21:14:32,824 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56beafa25bc4875b94bfe17ff9bcbc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=920, earliestPutTs=1733692470916 2024-12-08T21:14:32,824 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a31875bbdd48118f794ff55b12d51b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=920, earliestPutTs=1733692470916 2024-12-08T21:14:32,831 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#A#compaction#116 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:32,832 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/c2d76c86efb7423eb2bee98adf8a0fa7 is 50, key is test_row_0/A:col10/1733692470916/Put/seqid=0 2024-12-08T21:14:32,833 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#B#compaction#117 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:32,834 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/2dba1d87643c4157b2dc62b480393e75 is 50, key is test_row_0/B:col10/1733692470916/Put/seqid=0 2024-12-08T21:14:32,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741953_1129 (size=13595) 2024-12-08T21:14:32,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741954_1130 (size=13629) 2024-12-08T21:14:32,942 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-08T21:14:32,942 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:32,943 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:32,943 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:32,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:33,009 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-08T21:14:33,009 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:33,009 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:33,009 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:33,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-08T21:14:33,028 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:33,248 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/2dba1d87643c4157b2dc62b480393e75 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2dba1d87643c4157b2dc62b480393e75 2024-12-08T21:14:33,249 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/c2d76c86efb7423eb2bee98adf8a0fa7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/c2d76c86efb7423eb2bee98adf8a0fa7 2024-12-08T21:14:33,255 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/B of 0ef41e10da56e9b6857487fb3e409164 into 2dba1d87643c4157b2dc62b480393e75(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:33,255 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/A of 0ef41e10da56e9b6857487fb3e409164 into c2d76c86efb7423eb2bee98adf8a0fa7(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:33,255 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:33,255 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:33,255 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/A, priority=13, startTime=1733692472822; duration=0sec 2024-12-08T21:14:33,255 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/B, priority=13, startTime=1733692472822; duration=0sec 2024-12-08T21:14:33,256 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:33,256 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:33,256 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:A 2024-12-08T21:14:33,256 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:B 2024-12-08T21:14:33,256 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:33,257 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:33,257 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0ef41e10da56e9b6857487fb3e409164/C is initiating minor compaction (all files) 2024-12-08T21:14:33,257 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0ef41e10da56e9b6857487fb3e409164/C in TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:33,257 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/7bb201998ece4a6f89463b8eac41b037, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a757576ef9b0465a8e4a7141a510d4da, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e9225c3d6ac340759d97f28db1e6faef] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp, totalSize=34.8 K 2024-12-08T21:14:33,258 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb201998ece4a6f89463b8eac41b037, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=802, earliestPutTs=1733692468559 2024-12-08T21:14:33,258 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a757576ef9b0465a8e4a7141a510d4da, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=809, earliestPutTs=1733692468842 2024-12-08T21:14:33,259 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9225c3d6ac340759d97f28db1e6faef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=920, earliestPutTs=1733692470916 2024-12-08T21:14:33,267 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0ef41e10da56e9b6857487fb3e409164#C#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:33,268 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/3ec6508f16224f62bd04379f884ada50 is 50, key is test_row_0/C:col10/1733692470916/Put/seqid=0 2024-12-08T21:14:33,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741955_1131 (size=13629) 2024-12-08T21:14:33,770 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/3ec6508f16224f62bd04379f884ada50 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/3ec6508f16224f62bd04379f884ada50 2024-12-08T21:14:33,776 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0ef41e10da56e9b6857487fb3e409164/C of 0ef41e10da56e9b6857487fb3e409164 into 3ec6508f16224f62bd04379f884ada50(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:33,776 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0ef41e10da56e9b6857487fb3e409164: 2024-12-08T21:14:33,776 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., storeName=0ef41e10da56e9b6857487fb3e409164/C, priority=13, startTime=1733692472822; duration=0sec 2024-12-08T21:14:33,777 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:33,777 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0ef41e10da56e9b6857487fb3e409164:C 2024-12-08T21:14:35,281 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-08T21:14:35,281 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:14:35,281 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:35,281 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:35,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:35,281 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 256 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 214 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3996 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3670 2024-12-08T21:14:35,282 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T21:14:35,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1709 2024-12-08T21:14:35,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5126 rows 2024-12-08T21:14:35,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1711 2024-12-08T21:14:35,283 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5129 rows 2024-12-08T21:14:35,283 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:14:35,283 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:14:35,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:35,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:35,284 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:14:35,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T21:14:35,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T21:14:35,298 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692475297"}]},"ts":"1733692475297"} 2024-12-08T21:14:35,299 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T21:14:35,299 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T21:14:35,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:14:35,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, UNASSIGN}] 2024-12-08T21:14:35,306 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, UNASSIGN 2024-12-08T21:14:35,307 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0ef41e10da56e9b6857487fb3e409164, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:35,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, UNASSIGN because future has completed 2024-12-08T21:14:35,309 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:14:35,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:14:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T21:14:35,470 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:35,470 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:14:35,471 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 0ef41e10da56e9b6857487fb3e409164, disabling compactions & flushes 2024-12-08T21:14:35,471 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:35,471 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:35,471 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. after waiting 0 ms 2024-12-08T21:14:35,471 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:35,471 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 0ef41e10da56e9b6857487fb3e409164 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T21:14:35,472 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=A 2024-12-08T21:14:35,472 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:35,472 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=B 2024-12-08T21:14:35,472 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:35,472 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0ef41e10da56e9b6857487fb3e409164, store=C 2024-12-08T21:14:35,472 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:35,478 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/c77370472c0e4061b7fed4752acde71d is 50, key is test_row_0/A:col10/1733692473008/Put/seqid=0 2024-12-08T21:14:35,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741956_1132 (size=9857) 2024-12-08T21:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T21:14:35,885 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=929 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/c77370472c0e4061b7fed4752acde71d 2024-12-08T21:14:35,901 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/babc902ef8aa46ffba71ebafa6fb7f86 is 50, key is test_row_0/B:col10/1733692473008/Put/seqid=0 2024-12-08T21:14:35,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741957_1133 (size=9857) 2024-12-08T21:14:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T21:14:36,306 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=929 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/babc902ef8aa46ffba71ebafa6fb7f86 2024-12-08T21:14:36,314 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fec70e407b204bcc965f75f10d1faddc is 50, key is test_row_0/C:col10/1733692473008/Put/seqid=0 2024-12-08T21:14:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741958_1134 (size=9857) 2024-12-08T21:14:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T21:14:36,720 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=929 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fec70e407b204bcc965f75f10d1faddc 2024-12-08T21:14:36,731 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/A/c77370472c0e4061b7fed4752acde71d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/c77370472c0e4061b7fed4752acde71d 2024-12-08T21:14:36,737 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/c77370472c0e4061b7fed4752acde71d, entries=100, sequenceid=929, filesize=9.6 K 2024-12-08T21:14:36,738 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/B/babc902ef8aa46ffba71ebafa6fb7f86 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/babc902ef8aa46ffba71ebafa6fb7f86 2024-12-08T21:14:36,743 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/babc902ef8aa46ffba71ebafa6fb7f86, entries=100, sequenceid=929, filesize=9.6 K 2024-12-08T21:14:36,744 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/.tmp/C/fec70e407b204bcc965f75f10d1faddc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fec70e407b204bcc965f75f10d1faddc 2024-12-08T21:14:36,749 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fec70e407b204bcc965f75f10d1faddc, entries=100, sequenceid=929, filesize=9.6 K 2024-12-08T21:14:36,750 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0ef41e10da56e9b6857487fb3e409164 in 1279ms, sequenceid=929, compaction requested=false 2024-12-08T21:14:36,751 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/59a76afd5cab4d3395810577f611f2b1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/daccc3fbd4654b59937c0a1fcff1109b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5b9cd8e80b5c4087ab0e7f369f2ddff9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2bf895c68a854b868025b4d8320e9a64, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2353fc04ca5446d4bc83edd58f8dbe29, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8914968279b1401985c14c76b1b47da7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/926fedab226a41a5accb29ef3576bfff, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/cf3c84d517c14fa98db3b1eaafcf8d1b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5e7f51894740495ca78bc00d93147395, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/63bb382a14864b0ea70931a63b0b742a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e71346c764fd48aa9b9511361111a672, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e87f53e440194209b586ef0d46fbe71e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b9daa572965a449d8a62a0a359d48925, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/ab66001a4ff247339e58186f536beece, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/67bce704566c45938730b989a62e9706, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2fa8c58644844dac95b905f2a5d0a451, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/9484f0c43ac8451b8e7d25a999d24559, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8367fb52fef846919bbb185a5f9a3b66, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/91b64a6482674308ae6af99b0c490b78, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2480f3296a364fe4a20b18e83cb5ee7e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/02b8f4e645974bc8bb34f08012fa8a91, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/1950f94999404fa2b51fc46500586c00, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8557dc69a874484c8b037bfa493a4572, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/236a86643591474bb83af81ec50ab9cd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5cba7ca9d36f46c19d1436cf0f2a0bba, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7691987fdca04c3ba4c501d4da18e510, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e8afd791bc184ff886a62d1092417ca4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/0178b50ee2dd429e8116e17e189da759, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7b0e9ba8e2654a6fb0cd8585aa3cbaa9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e3f2531b60d8431689d613482b7dfb7f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b0f55e69118542339ea5c66444d08eec, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/367c2102bda8422f8993dcffb038d721, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/193479969ede44d18158359e312a0d02, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/6d6930db1f4849f3bd5d25ddb335d39e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/4775e651c9d84264ae654aa67777d8ed, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/242b9f2c53ac44a8a8ab80b0655260af, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/57d33786b1a64e099d78a6ea68cecea1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/a56beafa25bc4875b94bfe17ff9bcbc0] to archive 2024-12-08T21:14:36,755 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:14:36,760 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/59a76afd5cab4d3395810577f611f2b1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/59a76afd5cab4d3395810577f611f2b1 2024-12-08T21:14:36,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/daccc3fbd4654b59937c0a1fcff1109b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/daccc3fbd4654b59937c0a1fcff1109b 2024-12-08T21:14:36,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5b9cd8e80b5c4087ab0e7f369f2ddff9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5b9cd8e80b5c4087ab0e7f369f2ddff9 2024-12-08T21:14:36,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2bf895c68a854b868025b4d8320e9a64 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2bf895c68a854b868025b4d8320e9a64 2024-12-08T21:14:36,765 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2353fc04ca5446d4bc83edd58f8dbe29 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2353fc04ca5446d4bc83edd58f8dbe29 2024-12-08T21:14:36,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8914968279b1401985c14c76b1b47da7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8914968279b1401985c14c76b1b47da7 2024-12-08T21:14:36,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/926fedab226a41a5accb29ef3576bfff to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/926fedab226a41a5accb29ef3576bfff 2024-12-08T21:14:36,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/cf3c84d517c14fa98db3b1eaafcf8d1b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/cf3c84d517c14fa98db3b1eaafcf8d1b 2024-12-08T21:14:36,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5e7f51894740495ca78bc00d93147395 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5e7f51894740495ca78bc00d93147395 2024-12-08T21:14:36,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/63bb382a14864b0ea70931a63b0b742a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/63bb382a14864b0ea70931a63b0b742a 2024-12-08T21:14:36,772 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e71346c764fd48aa9b9511361111a672 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e71346c764fd48aa9b9511361111a672 2024-12-08T21:14:36,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e87f53e440194209b586ef0d46fbe71e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e87f53e440194209b586ef0d46fbe71e 2024-12-08T21:14:36,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b9daa572965a449d8a62a0a359d48925 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b9daa572965a449d8a62a0a359d48925 2024-12-08T21:14:36,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/ab66001a4ff247339e58186f536beece to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/ab66001a4ff247339e58186f536beece 2024-12-08T21:14:36,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/67bce704566c45938730b989a62e9706 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/67bce704566c45938730b989a62e9706 2024-12-08T21:14:36,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2fa8c58644844dac95b905f2a5d0a451 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2fa8c58644844dac95b905f2a5d0a451 2024-12-08T21:14:36,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/9484f0c43ac8451b8e7d25a999d24559 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/9484f0c43ac8451b8e7d25a999d24559 2024-12-08T21:14:36,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8367fb52fef846919bbb185a5f9a3b66 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8367fb52fef846919bbb185a5f9a3b66 2024-12-08T21:14:36,782 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/91b64a6482674308ae6af99b0c490b78 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/91b64a6482674308ae6af99b0c490b78 2024-12-08T21:14:36,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2480f3296a364fe4a20b18e83cb5ee7e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/2480f3296a364fe4a20b18e83cb5ee7e 2024-12-08T21:14:36,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/02b8f4e645974bc8bb34f08012fa8a91 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/02b8f4e645974bc8bb34f08012fa8a91 2024-12-08T21:14:36,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/1950f94999404fa2b51fc46500586c00 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/1950f94999404fa2b51fc46500586c00 2024-12-08T21:14:36,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8557dc69a874484c8b037bfa493a4572 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/8557dc69a874484c8b037bfa493a4572 2024-12-08T21:14:36,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/236a86643591474bb83af81ec50ab9cd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/236a86643591474bb83af81ec50ab9cd 2024-12-08T21:14:36,788 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5cba7ca9d36f46c19d1436cf0f2a0bba to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/5cba7ca9d36f46c19d1436cf0f2a0bba 2024-12-08T21:14:36,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7691987fdca04c3ba4c501d4da18e510 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7691987fdca04c3ba4c501d4da18e510 2024-12-08T21:14:36,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e8afd791bc184ff886a62d1092417ca4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e8afd791bc184ff886a62d1092417ca4 2024-12-08T21:14:36,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/0178b50ee2dd429e8116e17e189da759 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/0178b50ee2dd429e8116e17e189da759 2024-12-08T21:14:36,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7b0e9ba8e2654a6fb0cd8585aa3cbaa9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/7b0e9ba8e2654a6fb0cd8585aa3cbaa9 2024-12-08T21:14:36,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e3f2531b60d8431689d613482b7dfb7f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/e3f2531b60d8431689d613482b7dfb7f 2024-12-08T21:14:36,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b0f55e69118542339ea5c66444d08eec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/b0f55e69118542339ea5c66444d08eec 2024-12-08T21:14:36,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/367c2102bda8422f8993dcffb038d721 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/367c2102bda8422f8993dcffb038d721 2024-12-08T21:14:36,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/193479969ede44d18158359e312a0d02 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/193479969ede44d18158359e312a0d02 2024-12-08T21:14:36,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/6d6930db1f4849f3bd5d25ddb335d39e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/6d6930db1f4849f3bd5d25ddb335d39e 2024-12-08T21:14:36,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/4775e651c9d84264ae654aa67777d8ed to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/4775e651c9d84264ae654aa67777d8ed 2024-12-08T21:14:36,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/242b9f2c53ac44a8a8ab80b0655260af to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/242b9f2c53ac44a8a8ab80b0655260af 2024-12-08T21:14:36,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/57d33786b1a64e099d78a6ea68cecea1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/57d33786b1a64e099d78a6ea68cecea1 2024-12-08T21:14:36,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/a56beafa25bc4875b94bfe17ff9bcbc0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/a56beafa25bc4875b94bfe17ff9bcbc0 2024-12-08T21:14:36,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e3a405c00c4640b6b4037a51e6abacc7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/92c944acd26c41e88d82f439e656e04a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a5a2c1f914634e38a2cd05f3a228e4a9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6f2eef8893b341e59e43bd3d261437b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9da1fe5c06054239ade552accc6b6529, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3927a2c58699470783b03e4fda080510, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b783423c6e554df3af4e007aa96c055c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/f94d84f050e24d7d87dacdd0692767f4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2bd96241a38f42fea23ea0f6d66e7e19, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/5d16c4d639d147c1b07df3e5c1fbcd2f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a9b2f592e68a4ff5a94dfa238c927b9c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/409aafd0a8c24fa78563eaaaef1941aa, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3d86f5c907a4428d81d1907ea9c34926, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6b4f69354fb24651bc933be3b39caa84, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6e78abf71e94d26b23deb00eb836568, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b17df33404b146d5a2ac33e0b01b0fa7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/89722afe04e4425b9c561462ede3cf8c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/7a30542da1ee4818a1bf542a5c77319d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/310a8c4105cb4d469193e7608090a487, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/29426e9d84ec45658f53f668b8d7158a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6881758744f47388e762391389b4065, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c987c0b7584243c19866601002ee5b14, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8c6cdd6b3f2a4e09a6deec9ceaad5471, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4b9b68ca174046c69384c1122ab897c1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/64cde1237107484a990e8a83ce795bd6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/30552b4b361f48eeb6076096281e15a0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/01243b48587a4192bed6b240951f2d8f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8115a861217248639d05349c5d525d60, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/dea6fb29131c430da5b203aa9e366832, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/1d78a94b76334df49311d87734e9a034, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9827ab06318d449ea16f214fb52b0a08, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9723b26e383644beb927eb76af8e43a6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c4aae59ea02e40129b1919de4524d263, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/16fce10029e142639c9131c80ba7423d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4225a1686a734ea69bce7bc3ae83c86b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e83b234a7fff44ce8abf39c467a5ff5c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/97ac242636d246ca8b0bfde76f10f652, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/21099320c5b14a49b799ed463f0c4a1d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/88a31875bbdd48118f794ff55b12d51b] to archive 2024-12-08T21:14:36,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:14:36,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e3a405c00c4640b6b4037a51e6abacc7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e3a405c00c4640b6b4037a51e6abacc7 2024-12-08T21:14:36,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/92c944acd26c41e88d82f439e656e04a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/92c944acd26c41e88d82f439e656e04a 2024-12-08T21:14:36,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a5a2c1f914634e38a2cd05f3a228e4a9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a5a2c1f914634e38a2cd05f3a228e4a9 2024-12-08T21:14:36,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6f2eef8893b341e59e43bd3d261437b7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6f2eef8893b341e59e43bd3d261437b7 2024-12-08T21:14:36,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9da1fe5c06054239ade552accc6b6529 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9da1fe5c06054239ade552accc6b6529 2024-12-08T21:14:36,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3927a2c58699470783b03e4fda080510 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3927a2c58699470783b03e4fda080510 2024-12-08T21:14:36,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b783423c6e554df3af4e007aa96c055c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b783423c6e554df3af4e007aa96c055c 2024-12-08T21:14:36,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/f94d84f050e24d7d87dacdd0692767f4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/f94d84f050e24d7d87dacdd0692767f4 2024-12-08T21:14:36,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2bd96241a38f42fea23ea0f6d66e7e19 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2bd96241a38f42fea23ea0f6d66e7e19 2024-12-08T21:14:36,823 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/5d16c4d639d147c1b07df3e5c1fbcd2f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/5d16c4d639d147c1b07df3e5c1fbcd2f 2024-12-08T21:14:36,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a9b2f592e68a4ff5a94dfa238c927b9c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/a9b2f592e68a4ff5a94dfa238c927b9c 2024-12-08T21:14:36,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/409aafd0a8c24fa78563eaaaef1941aa to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/409aafd0a8c24fa78563eaaaef1941aa 2024-12-08T21:14:36,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3d86f5c907a4428d81d1907ea9c34926 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/3d86f5c907a4428d81d1907ea9c34926 2024-12-08T21:14:36,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6b4f69354fb24651bc933be3b39caa84 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/6b4f69354fb24651bc933be3b39caa84 2024-12-08T21:14:36,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6e78abf71e94d26b23deb00eb836568 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6e78abf71e94d26b23deb00eb836568 2024-12-08T21:14:36,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b17df33404b146d5a2ac33e0b01b0fa7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/b17df33404b146d5a2ac33e0b01b0fa7 2024-12-08T21:14:36,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/89722afe04e4425b9c561462ede3cf8c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/89722afe04e4425b9c561462ede3cf8c 2024-12-08T21:14:36,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/7a30542da1ee4818a1bf542a5c77319d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/7a30542da1ee4818a1bf542a5c77319d 2024-12-08T21:14:36,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/310a8c4105cb4d469193e7608090a487 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/310a8c4105cb4d469193e7608090a487 2024-12-08T21:14:36,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/29426e9d84ec45658f53f668b8d7158a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/29426e9d84ec45658f53f668b8d7158a 2024-12-08T21:14:36,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6881758744f47388e762391389b4065 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c6881758744f47388e762391389b4065 2024-12-08T21:14:36,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c987c0b7584243c19866601002ee5b14 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c987c0b7584243c19866601002ee5b14 2024-12-08T21:14:36,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8c6cdd6b3f2a4e09a6deec9ceaad5471 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8c6cdd6b3f2a4e09a6deec9ceaad5471 2024-12-08T21:14:36,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4b9b68ca174046c69384c1122ab897c1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4b9b68ca174046c69384c1122ab897c1 2024-12-08T21:14:36,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/64cde1237107484a990e8a83ce795bd6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/64cde1237107484a990e8a83ce795bd6 2024-12-08T21:14:36,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/30552b4b361f48eeb6076096281e15a0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/30552b4b361f48eeb6076096281e15a0 2024-12-08T21:14:36,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/01243b48587a4192bed6b240951f2d8f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/01243b48587a4192bed6b240951f2d8f 2024-12-08T21:14:36,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8115a861217248639d05349c5d525d60 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/8115a861217248639d05349c5d525d60 2024-12-08T21:14:36,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/dea6fb29131c430da5b203aa9e366832 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/dea6fb29131c430da5b203aa9e366832 2024-12-08T21:14:36,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/1d78a94b76334df49311d87734e9a034 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/1d78a94b76334df49311d87734e9a034 2024-12-08T21:14:36,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9827ab06318d449ea16f214fb52b0a08 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9827ab06318d449ea16f214fb52b0a08 2024-12-08T21:14:36,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9723b26e383644beb927eb76af8e43a6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/9723b26e383644beb927eb76af8e43a6 2024-12-08T21:14:36,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c4aae59ea02e40129b1919de4524d263 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/c4aae59ea02e40129b1919de4524d263 2024-12-08T21:14:36,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/16fce10029e142639c9131c80ba7423d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/16fce10029e142639c9131c80ba7423d 2024-12-08T21:14:36,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4225a1686a734ea69bce7bc3ae83c86b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/4225a1686a734ea69bce7bc3ae83c86b 2024-12-08T21:14:36,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e83b234a7fff44ce8abf39c467a5ff5c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/e83b234a7fff44ce8abf39c467a5ff5c 2024-12-08T21:14:36,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/97ac242636d246ca8b0bfde76f10f652 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/97ac242636d246ca8b0bfde76f10f652 2024-12-08T21:14:36,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/21099320c5b14a49b799ed463f0c4a1d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/21099320c5b14a49b799ed463f0c4a1d 2024-12-08T21:14:36,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/88a31875bbdd48118f794ff55b12d51b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/88a31875bbdd48118f794ff55b12d51b 2024-12-08T21:14:36,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/48be02c5443147259ae6b60d3a03b844, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/99cb931d59af458f9f64cc562c6f5077, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/241bec018e3b4f4f9fc468c1a7d64b0a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/0dcabe26512c4a5ea42f50fbbff0892b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a9046c9334eb4d86940a0f626e98987b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fe25c52d5b444bbe959e791ca5196528, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c181926030a241f7bfb009b0cdd01fba, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42c6bc4d268b42dfa0be082336586a36, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3f2b8bafaf74f42a0870d33407c93b3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fb9244a840fc41b5b27fece4fe75f340, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/64cbafc999694111939b183b3dfe23f2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/cf3f80aadda646368129d282f393297c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a2246e3ff94497bfaca5a0adf96adf, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a76e02fcd94543bbb57766ca48ff204c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/ab9b775c303943b7b06a1cf67515b2e5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8b8a65c949e045e985132665996f3c71, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42d80fcd50534c489df0960e716965a1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94e57e248fee4cbf8a4ec4c19d9510bc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a1c6db921d48148d07406cc21448fc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/39a5da8de49f4d7ea4bcf6ad69e12565, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/6644bae8cd5547e5bf2cabba5157efae, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/750ccca067bd497f811ed5517470c54b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c7ddc80c24ec48839d91eda710f43db8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94f264571b3346f892eeb9f179870efb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c5271671995c4e909034ad940b9b98c5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/02ad3f102a90418184611bff8898f38a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/aff642c3104e4c8e83d352696be221c0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/f4d8fb0de1f3458abfda1e9b23a9143a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/b3319e886cb44f37ae22c3e9fd791779, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e78b5ad8c08148679b545cbd23550e0b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8afb60ca19d64109844805e6de5154e6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/16ec566f771e458da3120fb8718e5df0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/5f9119d873b94987a549eb9027471721, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8eac0e3b256e4fa783c206c33c06b9e5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/9d66d4adea494179a5c2e9308f911453, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/7bb201998ece4a6f89463b8eac41b037, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/54798e2505fa40e38856c734caf9af2b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a757576ef9b0465a8e4a7141a510d4da, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e9225c3d6ac340759d97f28db1e6faef] to archive 2024-12-08T21:14:36,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:14:36,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/48be02c5443147259ae6b60d3a03b844 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/48be02c5443147259ae6b60d3a03b844 2024-12-08T21:14:36,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/99cb931d59af458f9f64cc562c6f5077 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/99cb931d59af458f9f64cc562c6f5077 2024-12-08T21:14:36,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/241bec018e3b4f4f9fc468c1a7d64b0a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/241bec018e3b4f4f9fc468c1a7d64b0a 2024-12-08T21:14:36,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/0dcabe26512c4a5ea42f50fbbff0892b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/0dcabe26512c4a5ea42f50fbbff0892b 2024-12-08T21:14:36,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a9046c9334eb4d86940a0f626e98987b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a9046c9334eb4d86940a0f626e98987b 2024-12-08T21:14:36,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fe25c52d5b444bbe959e791ca5196528 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fe25c52d5b444bbe959e791ca5196528 2024-12-08T21:14:36,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c181926030a241f7bfb009b0cdd01fba to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c181926030a241f7bfb009b0cdd01fba 2024-12-08T21:14:36,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42c6bc4d268b42dfa0be082336586a36 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42c6bc4d268b42dfa0be082336586a36 2024-12-08T21:14:36,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3f2b8bafaf74f42a0870d33407c93b3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3f2b8bafaf74f42a0870d33407c93b3 2024-12-08T21:14:36,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fb9244a840fc41b5b27fece4fe75f340 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fb9244a840fc41b5b27fece4fe75f340 2024-12-08T21:14:36,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/64cbafc999694111939b183b3dfe23f2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/64cbafc999694111939b183b3dfe23f2 2024-12-08T21:14:36,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/cf3f80aadda646368129d282f393297c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/cf3f80aadda646368129d282f393297c 2024-12-08T21:14:36,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a2246e3ff94497bfaca5a0adf96adf to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a2246e3ff94497bfaca5a0adf96adf 2024-12-08T21:14:36,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a76e02fcd94543bbb57766ca48ff204c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a76e02fcd94543bbb57766ca48ff204c 2024-12-08T21:14:36,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/ab9b775c303943b7b06a1cf67515b2e5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/ab9b775c303943b7b06a1cf67515b2e5 2024-12-08T21:14:36,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8b8a65c949e045e985132665996f3c71 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8b8a65c949e045e985132665996f3c71 2024-12-08T21:14:36,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42d80fcd50534c489df0960e716965a1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/42d80fcd50534c489df0960e716965a1 2024-12-08T21:14:36,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94e57e248fee4cbf8a4ec4c19d9510bc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94e57e248fee4cbf8a4ec4c19d9510bc 2024-12-08T21:14:36,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a1c6db921d48148d07406cc21448fc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e3a1c6db921d48148d07406cc21448fc 2024-12-08T21:14:36,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/39a5da8de49f4d7ea4bcf6ad69e12565 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/39a5da8de49f4d7ea4bcf6ad69e12565 2024-12-08T21:14:36,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/6644bae8cd5547e5bf2cabba5157efae to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/6644bae8cd5547e5bf2cabba5157efae 2024-12-08T21:14:36,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/750ccca067bd497f811ed5517470c54b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/750ccca067bd497f811ed5517470c54b 2024-12-08T21:14:36,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c7ddc80c24ec48839d91eda710f43db8 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c7ddc80c24ec48839d91eda710f43db8 2024-12-08T21:14:36,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94f264571b3346f892eeb9f179870efb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/94f264571b3346f892eeb9f179870efb 2024-12-08T21:14:36,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c5271671995c4e909034ad940b9b98c5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/c5271671995c4e909034ad940b9b98c5 2024-12-08T21:14:36,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/02ad3f102a90418184611bff8898f38a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/02ad3f102a90418184611bff8898f38a 2024-12-08T21:14:36,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/aff642c3104e4c8e83d352696be221c0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/aff642c3104e4c8e83d352696be221c0 2024-12-08T21:14:36,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/f4d8fb0de1f3458abfda1e9b23a9143a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/f4d8fb0de1f3458abfda1e9b23a9143a 2024-12-08T21:14:36,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/b3319e886cb44f37ae22c3e9fd791779 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/b3319e886cb44f37ae22c3e9fd791779 2024-12-08T21:14:36,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e78b5ad8c08148679b545cbd23550e0b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e78b5ad8c08148679b545cbd23550e0b 2024-12-08T21:14:36,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8afb60ca19d64109844805e6de5154e6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8afb60ca19d64109844805e6de5154e6 2024-12-08T21:14:36,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/16ec566f771e458da3120fb8718e5df0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/16ec566f771e458da3120fb8718e5df0 2024-12-08T21:14:36,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/5f9119d873b94987a549eb9027471721 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/5f9119d873b94987a549eb9027471721 2024-12-08T21:14:36,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8eac0e3b256e4fa783c206c33c06b9e5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/8eac0e3b256e4fa783c206c33c06b9e5 2024-12-08T21:14:36,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/9d66d4adea494179a5c2e9308f911453 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/9d66d4adea494179a5c2e9308f911453 2024-12-08T21:14:36,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/7bb201998ece4a6f89463b8eac41b037 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/7bb201998ece4a6f89463b8eac41b037 2024-12-08T21:14:36,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/54798e2505fa40e38856c734caf9af2b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/54798e2505fa40e38856c734caf9af2b 2024-12-08T21:14:36,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a757576ef9b0465a8e4a7141a510d4da to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/a757576ef9b0465a8e4a7141a510d4da 2024-12-08T21:14:36,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e9225c3d6ac340759d97f28db1e6faef to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/e9225c3d6ac340759d97f28db1e6faef 2024-12-08T21:14:36,903 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/recovered.edits/932.seqid, newMaxSeqId=932, maxSeqId=1 2024-12-08T21:14:36,904 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. 2024-12-08T21:14:36,904 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 0ef41e10da56e9b6857487fb3e409164: Waiting for close lock at 1733692475471Running coprocessor pre-close hooks at 1733692475471Disabling compacts and flushes for region at 1733692475471Disabling writes for close at 1733692475471Obtaining lock to block concurrent updates at 1733692475471Preparing flush snapshotting stores in 0ef41e10da56e9b6857487fb3e409164 at 1733692475471Finished memstore snapshotting TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733692475472 (+1 ms)Flushing stores of TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164. at 1733692475473 (+1 ms)Flushing 0ef41e10da56e9b6857487fb3e409164/A: creating writer at 1733692475473Flushing 0ef41e10da56e9b6857487fb3e409164/A: appending metadata at 1733692475478 (+5 ms)Flushing 0ef41e10da56e9b6857487fb3e409164/A: closing flushed file at 1733692475478Flushing 0ef41e10da56e9b6857487fb3e409164/B: creating writer at 1733692475897 (+419 ms)Flushing 0ef41e10da56e9b6857487fb3e409164/B: appending metadata at 1733692475900 (+3 ms)Flushing 0ef41e10da56e9b6857487fb3e409164/B: closing flushed file at 1733692475901 (+1 ms)Flushing 0ef41e10da56e9b6857487fb3e409164/C: creating writer at 1733692476310 (+409 ms)Flushing 0ef41e10da56e9b6857487fb3e409164/C: appending metadata at 1733692476313 (+3 ms)Flushing 0ef41e10da56e9b6857487fb3e409164/C: closing flushed file at 1733692476313Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1ba5c4: reopening flushed file at 1733692476729 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a57c511: reopening flushed file at 1733692476737 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@443422a6: reopening flushed file at 1733692476744 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 0ef41e10da56e9b6857487fb3e409164 in 1279ms, sequenceid=929, compaction requested=false at 1733692476750 (+6 ms)Writing region close event to WAL at 1733692476900 (+150 ms)Running coprocessor post-close hooks at 1733692476904 (+4 ms)Closed at 1733692476904 2024-12-08T21:14:36,907 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:36,907 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0ef41e10da56e9b6857487fb3e409164, regionState=CLOSED 2024-12-08T21:14:36,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:14:36,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-08T21:14:36,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 0ef41e10da56e9b6857487fb3e409164, server=e653e208f502,38457,1733692444075 in 1.6010 sec 2024-12-08T21:14:36,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-08T21:14:36,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0ef41e10da56e9b6857487fb3e409164, UNASSIGN in 1.6060 sec 2024-12-08T21:14:36,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-08T21:14:36,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6130 sec 2024-12-08T21:14:36,916 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692476916"}]},"ts":"1733692476916"} 2024-12-08T21:14:36,918 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T21:14:36,918 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T21:14:36,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6260 sec 2024-12-08T21:14:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-08T21:14:37,458 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:37,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T21:14:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:37,471 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T21:14:37,473 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:37,477 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:37,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/recovered.edits] 2024-12-08T21:14:37,484 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/c2d76c86efb7423eb2bee98adf8a0fa7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/c2d76c86efb7423eb2bee98adf8a0fa7 2024-12-08T21:14:37,486 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/c77370472c0e4061b7fed4752acde71d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/A/c77370472c0e4061b7fed4752acde71d 2024-12-08T21:14:37,488 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2dba1d87643c4157b2dc62b480393e75 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/2dba1d87643c4157b2dc62b480393e75 2024-12-08T21:14:37,488 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/babc902ef8aa46ffba71ebafa6fb7f86 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/B/babc902ef8aa46ffba71ebafa6fb7f86 2024-12-08T21:14:37,490 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/3ec6508f16224f62bd04379f884ada50 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/3ec6508f16224f62bd04379f884ada50 2024-12-08T21:14:37,491 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fec70e407b204bcc965f75f10d1faddc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/C/fec70e407b204bcc965f75f10d1faddc 2024-12-08T21:14:37,493 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/recovered.edits/932.seqid to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164/recovered.edits/932.seqid 2024-12-08T21:14:37,494 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0ef41e10da56e9b6857487fb3e409164 2024-12-08T21:14:37,494 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T21:14:37,498 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:37,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-08T21:14:37,507 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T21:14:37,535 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T21:14:37,536 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:37,537 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T21:14:37,537 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733692477537"}]},"ts":"9223372036854775807"} 2024-12-08T21:14:37,541 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T21:14:37,541 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0ef41e10da56e9b6857487fb3e409164, NAME => 'TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T21:14:37,542 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T21:14:37,542 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733692477542"}]},"ts":"9223372036854775807"} 2024-12-08T21:14:37,546 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T21:14:37,547 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:37,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-12-08T21:14:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-08T21:14:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T21:14:37,578 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:37,589 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e653e208f502:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1746726969_22 at /127.0.0.1:43962 [Waiting for operation #128] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-890891175_22 at /127.0.0.1:36866 [Waiting for operation #321] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-890891175_22 at /127.0.0.1:43424 [Waiting for operation #310] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e653e208f502:38457-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1746726969_22 at /127.0.0.1:57804 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e653e208f502:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=292 (was 234) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7766 (was 8391) 2024-12-08T21:14:37,598 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=292, ProcessCount=11, AvailableMemoryMB=7765 2024-12-08T21:14:37,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:14:37,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:14:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:37,602 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:14:37,602 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:37,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-12-08T21:14:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T21:14:37,603 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:14:37,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741959_1135 (size=1061) 2024-12-08T21:14:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T21:14:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T21:14:38,014 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:14:38,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741960_1136 (size=53) 2024-12-08T21:14:38,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T21:14:38,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:38,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cddc5d594981d66d83e5fc7c39ab45c4, disabling compactions & flushes 2024-12-08T21:14:38,426 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:38,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:38,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. after waiting 0 ms 2024-12-08T21:14:38,426 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:38,426 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:38,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cddc5d594981d66d83e5fc7c39ab45c4: Waiting for close lock at 1733692478426Disabling compacts and flushes for region at 1733692478426Disabling writes for close at 1733692478426Writing region close event to WAL at 1733692478426Closed at 1733692478426 2024-12-08T21:14:38,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:14:38,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733692478429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733692478429"}]},"ts":"1733692478429"} 2024-12-08T21:14:38,432 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:14:38,434 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:14:38,434 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692478434"}]},"ts":"1733692478434"} 2024-12-08T21:14:38,437 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T21:14:38,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, ASSIGN}] 2024-12-08T21:14:38,439 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, ASSIGN 2024-12-08T21:14:38,440 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, ASSIGN; state=OFFLINE, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=false 2024-12-08T21:14:38,591 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:38,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, ASSIGN because future has completed 2024-12-08T21:14:38,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:14:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T21:14:38,971 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:38,971 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:14:38,972 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,973 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:38,973 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,973 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,976 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,978 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:38,980 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cddc5d594981d66d83e5fc7c39ab45c4 columnFamilyName A 2024-12-08T21:14:38,980 DEBUG [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:38,980 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(327): Store=cddc5d594981d66d83e5fc7c39ab45c4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:38,981 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,982 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:38,982 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cddc5d594981d66d83e5fc7c39ab45c4 columnFamilyName B 2024-12-08T21:14:38,983 DEBUG [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:38,983 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(327): Store=cddc5d594981d66d83e5fc7c39ab45c4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:38,983 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,984 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:38,985 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cddc5d594981d66d83e5fc7c39ab45c4 columnFamilyName C 2024-12-08T21:14:38,985 DEBUG [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:38,985 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(327): Store=cddc5d594981d66d83e5fc7c39ab45c4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:38,986 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:38,986 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,987 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,987 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,989 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,989 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,990 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:14:38,992 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,995 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:14:38,996 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened cddc5d594981d66d83e5fc7c39ab45c4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73870489, jitterRate=0.10075606405735016}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:14:38,996 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:38,997 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for cddc5d594981d66d83e5fc7c39ab45c4: Running coprocessor pre-open hook at 1733692478973Writing region info on filesystem at 1733692478973Initializing all the Stores at 1733692478975 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692478976 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692478976Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692478976Cleaning up temporary data from old regions at 1733692478989 (+13 ms)Running coprocessor post-open hooks at 1733692478996 (+7 ms)Region opened successfully at 1733692478997 (+1 ms) 2024-12-08T21:14:38,998 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., pid=34, masterSystemTime=1733692478750 2024-12-08T21:14:39,000 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:39,001 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:39,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=OPEN, openSeqNum=2, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:39,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:14:39,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-08T21:14:39,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 in 407 msec 2024-12-08T21:14:39,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-08T21:14:39,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, ASSIGN in 569 msec 2024-12-08T21:14:39,010 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:14:39,010 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692479010"}]},"ts":"1733692479010"} 2024-12-08T21:14:39,011 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T21:14:39,012 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:14:39,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.4120 sec 2024-12-08T21:14:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-08T21:14:39,749 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:39,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:39,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:39,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:39,755 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:39,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa6213a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:39,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b24555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:39,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:39,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:39,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:39,764 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:14:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e653e208f502,45981,1733692443391 2024-12-08T21:14:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bf6b1e5 2024-12-08T21:14:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:14:39,766 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:14:39,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:14:39,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:14:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T21:14:39,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741961_1137 (size=1097) 2024-12-08T21:14:40,187 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T21:14:40,187 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T21:14:40,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:14:40,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, REOPEN/MOVE}] 2024-12-08T21:14:40,198 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, REOPEN/MOVE 2024-12-08T21:14:40,199 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, REOPEN/MOVE because future has completed 2024-12-08T21:14:40,201 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:14:40,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:14:40,355 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,355 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:14:40,355 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing cddc5d594981d66d83e5fc7c39ab45c4, disabling compactions & flushes 2024-12-08T21:14:40,356 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,356 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,356 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. after waiting 0 ms 2024-12-08T21:14:40,356 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,365 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T21:14:40,366 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,366 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for cddc5d594981d66d83e5fc7c39ab45c4: Waiting for close lock at 1733692480355Running coprocessor pre-close hooks at 1733692480355Disabling compacts and flushes for region at 1733692480355Disabling writes for close at 1733692480356 (+1 ms)Writing region close event to WAL at 1733692480357 (+1 ms)Running coprocessor post-close hooks at 1733692480366 (+9 ms)Closed at 1733692480366 2024-12-08T21:14:40,366 WARN [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: cddc5d594981d66d83e5fc7c39ab45c4 to self. 2024-12-08T21:14:40,369 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,369 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=CLOSED 2024-12-08T21:14:40,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:14:40,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-08T21:14:40,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 in 172 msec 2024-12-08T21:14:40,376 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, REOPEN/MOVE; state=CLOSED, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=true 2024-12-08T21:14:40,527 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, REOPEN/MOVE because future has completed 2024-12-08T21:14:40,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:14:40,691 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,691 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:14:40,692 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,692 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:14:40,692 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,692 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,696 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,697 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:40,701 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cddc5d594981d66d83e5fc7c39ab45c4 columnFamilyName A 2024-12-08T21:14:40,703 DEBUG [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:40,703 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(327): Store=cddc5d594981d66d83e5fc7c39ab45c4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:40,704 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,704 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:40,705 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cddc5d594981d66d83e5fc7c39ab45c4 columnFamilyName B 2024-12-08T21:14:40,705 DEBUG [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:40,705 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(327): Store=cddc5d594981d66d83e5fc7c39ab45c4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:40,705 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,706 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:14:40,706 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cddc5d594981d66d83e5fc7c39ab45c4 columnFamilyName C 2024-12-08T21:14:40,706 DEBUG [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:40,707 INFO [StoreOpener-cddc5d594981d66d83e5fc7c39ab45c4-1 {}] regionserver.HStore(327): Store=cddc5d594981d66d83e5fc7c39ab45c4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:14:40,707 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,707 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,708 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,709 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,710 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,710 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,710 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:14:40,711 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,712 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened cddc5d594981d66d83e5fc7c39ab45c4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65755046, jitterRate=-0.020173460245132446}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:14:40,712 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,713 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for cddc5d594981d66d83e5fc7c39ab45c4: Running coprocessor pre-open hook at 1733692480692Writing region info on filesystem at 1733692480692Initializing all the Stores at 1733692480694 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692480694Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692480694Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692480694Cleaning up temporary data from old regions at 1733692480710 (+16 ms)Running coprocessor post-open hooks at 1733692480712 (+2 ms)Region opened successfully at 1733692480713 (+1 ms) 2024-12-08T21:14:40,713 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., pid=39, masterSystemTime=1733692480684 2024-12-08T21:14:40,715 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,715 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,716 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=OPEN, openSeqNum=5, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:14:40,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-12-08T21:14:40,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 in 189 msec 2024-12-08T21:14:40,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-08T21:14:40,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, REOPEN/MOVE in 524 msec 2024-12-08T21:14:40,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-08T21:14:40,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-12-08T21:14:40,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-12-08T21:14:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-08T21:14:40,733 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:40,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,735 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33512826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,737 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289e5f6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,739 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,741 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39300d76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,744 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e5c5ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,745 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253f221d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,748 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79145b8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,750 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694ee1e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b9f84c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24d86000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,756 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66daea54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb3dbde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173af643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,761 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e409a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,763 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0ba3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,765 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e0e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:14:40,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:14:40,767 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:14:40,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:14:40,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b58318f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:14:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:14:40,769 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:14:40,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:14:40,770 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,770 DEBUG [Thread-693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,770 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,770 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,771 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,771 DEBUG [Thread-684 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,772 DEBUG [Thread-684 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,772 DEBUG [Thread-691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,772 DEBUG [Thread-695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,773 DEBUG [Thread-689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-08T21:14:40,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T21:14:40,775 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:40,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,777 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:40,777 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:40,777 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,778 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:14:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:14:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:14:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:40,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:14:40,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:40,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:40,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:40,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120851540a04cd3e4fbb9088e3590fc3f9e2_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692480790/Put/seqid=0 2024-12-08T21:14:40,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741962_1138 (size=12154) 2024-12-08T21:14:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T21:14:40,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692480875, max timestamp=1733692480887Num uniques 859; 2024-12-08T21:14:40,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692480875, max timestamp=1733692480887Num uniques 859; 2024-12-08T21:14:40,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692480875, max timestamp=1733692480887Num uniques 859; 2024-12-08T21:14:40,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:40,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:40,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:14:40,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692540915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692540916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:40,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692540926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T21:14:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:40,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:40,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:40,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692540926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692540927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692541038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692541038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692541039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692541039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692541040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T21:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T21:14:41,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T21:14:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692541248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692541248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692541248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692541248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692541248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:41,261 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120851540a04cd3e4fbb9088e3590fc3f9e2_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120851540a04cd3e4fbb9088e3590fc3f9e2_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:41,262 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e0f9bdc9acc548c79a044d12d3644a1e, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:41,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e0f9bdc9acc548c79a044d12d3644a1e is 175, key is test_row_0/A:col10/1733692480790/Put/seqid=0 2024-12-08T21:14:41,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741963_1139 (size=30955) 2024-12-08T21:14:41,293 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e0f9bdc9acc548c79a044d12d3644a1e 2024-12-08T21:14:41,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/e362f557e7be40b1952aba5729837260 is 50, key is test_row_0/B:col10/1733692480790/Put/seqid=0 2024-12-08T21:14:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741964_1140 (size=12001) 2024-12-08T21:14:41,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/e362f557e7be40b1952aba5729837260 2024-12-08T21:14:41,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/dfa7f77d079349d69dcf1be962d79693 is 50, key is test_row_0/C:col10/1733692480790/Put/seqid=0 2024-12-08T21:14:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741965_1141 (size=12001) 2024-12-08T21:14:41,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T21:14:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:41,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T21:14:41,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T21:14:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692541558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692541558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692541558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692541558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692541559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:41,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T21:14:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:41,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:41,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/dfa7f77d079349d69dcf1be962d79693 2024-12-08T21:14:41,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e0f9bdc9acc548c79a044d12d3644a1e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e0f9bdc9acc548c79a044d12d3644a1e 2024-12-08T21:14:41,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e0f9bdc9acc548c79a044d12d3644a1e, entries=150, sequenceid=14, filesize=30.2 K 2024-12-08T21:14:41,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/e362f557e7be40b1952aba5729837260 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/e362f557e7be40b1952aba5729837260 2024-12-08T21:14:41,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/e362f557e7be40b1952aba5729837260, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T21:14:41,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/dfa7f77d079349d69dcf1be962d79693 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/dfa7f77d079349d69dcf1be962d79693 2024-12-08T21:14:41,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/dfa7f77d079349d69dcf1be962d79693, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T21:14:41,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for cddc5d594981d66d83e5fc7c39ab45c4 in 994ms, sequenceid=14, compaction requested=false 2024-12-08T21:14:41,807 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-08T21:14:41,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:41,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-08T21:14:41,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:41,855 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-08T21:14:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:41,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084155c28dba274776a6d12407e1af888e_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692480912/Put/seqid=0 2024-12-08T21:14:41,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741966_1142 (size=12154) 2024-12-08T21:14:41,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:41,897 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084155c28dba274776a6d12407e1af888e_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084155c28dba274776a6d12407e1af888e_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:41,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/4e08eb2502ca44e6a214ae120e94e145, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:41,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/4e08eb2502ca44e6a214ae120e94e145 is 175, key is test_row_0/A:col10/1733692480912/Put/seqid=0 2024-12-08T21:14:41,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741967_1143 (size=30955) 2024-12-08T21:14:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T21:14:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:42,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:42,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692542076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692542077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692542078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692542079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692542080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692542188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692542188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692542189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692542189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,308 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/4e08eb2502ca44e6a214ae120e94e145 2024-12-08T21:14:42,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/8f0aaa426d8d4006886c7faf1719982c is 50, key is test_row_0/B:col10/1733692480912/Put/seqid=0 2024-12-08T21:14:42,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741968_1144 (size=12001) 2024-12-08T21:14:42,338 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/8f0aaa426d8d4006886c7faf1719982c 2024-12-08T21:14:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5622d57b10704d4d8e0f822c2e67a256 is 50, key is test_row_0/C:col10/1733692480912/Put/seqid=0 2024-12-08T21:14:42,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741969_1145 (size=12001) 2024-12-08T21:14:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692542398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692542398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692542399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692542399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,435 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:14:42,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692542708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692542708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692542709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692542718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:42,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:42,758 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5622d57b10704d4d8e0f822c2e67a256 2024-12-08T21:14:42,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/4e08eb2502ca44e6a214ae120e94e145 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/4e08eb2502ca44e6a214ae120e94e145 2024-12-08T21:14:42,773 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/4e08eb2502ca44e6a214ae120e94e145, entries=150, sequenceid=51, filesize=30.2 K 2024-12-08T21:14:42,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/8f0aaa426d8d4006886c7faf1719982c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8f0aaa426d8d4006886c7faf1719982c 2024-12-08T21:14:42,782 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8f0aaa426d8d4006886c7faf1719982c, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T21:14:42,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5622d57b10704d4d8e0f822c2e67a256 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5622d57b10704d4d8e0f822c2e67a256 2024-12-08T21:14:42,790 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5622d57b10704d4d8e0f822c2e67a256, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T21:14:42,792 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for cddc5d594981d66d83e5fc7c39ab45c4 in 938ms, sequenceid=51, compaction requested=false 2024-12-08T21:14:42,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:42,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-08T21:14:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-08T21:14:42,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-08T21:14:42,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0200 sec 2024-12-08T21:14:42,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.0290 sec 2024-12-08T21:14:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-08T21:14:42,918 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:42,919 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-08T21:14:42,921 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T21:14:42,922 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:42,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T21:14:43,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-08T21:14:43,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:43,078 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:14:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:43,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c6171b9f4b1f4812b0ead6a09f108b93_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692482078/Put/seqid=0 2024-12-08T21:14:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:43,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:43,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741970_1146 (size=12154) 2024-12-08T21:14:43,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692483186, max timestamp=1733692483213Num uniques 859; 2024-12-08T21:14:43,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692483186, max timestamp=1733692483213Num uniques 859; 2024-12-08T21:14:43,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692483186, max timestamp=1733692483213Num uniques 859; 2024-12-08T21:14:43,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:43,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:43,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T21:14:43,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692543270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692543272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692543274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692543277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692543284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692543388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692543388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692543398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692543399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692543400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:43,546 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c6171b9f4b1f4812b0ead6a09f108b93_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c6171b9f4b1f4812b0ead6a09f108b93_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T21:14:43,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/a2eec19589414799bac341e2322f1857, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:43,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/a2eec19589414799bac341e2322f1857 is 175, key is test_row_0/A:col10/1733692482078/Put/seqid=0 2024-12-08T21:14:43,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741971_1147 (size=30955) 2024-12-08T21:14:43,598 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/a2eec19589414799bac341e2322f1857 2024-12-08T21:14:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692543608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692543608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692543608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/c4e001cebd884f3f9f9b9c19fb698303 is 50, key is test_row_0/B:col10/1733692482078/Put/seqid=0 2024-12-08T21:14:43,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692543620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692543620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741972_1148 (size=12001) 2024-12-08T21:14:43,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692543917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692543917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692543918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692543928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692543928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T21:14:44,059 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/c4e001cebd884f3f9f9b9c19fb698303 2024-12-08T21:14:44,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5d0b95429cf74e67bba23f984f00e06e is 50, key is test_row_0/C:col10/1733692482078/Put/seqid=0 2024-12-08T21:14:44,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741973_1149 (size=12001) 2024-12-08T21:14:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692544427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692544427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692544428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692544438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692544438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:44,492 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5d0b95429cf74e67bba23f984f00e06e 2024-12-08T21:14:44,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/a2eec19589414799bac341e2322f1857 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a2eec19589414799bac341e2322f1857 2024-12-08T21:14:44,507 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a2eec19589414799bac341e2322f1857, entries=150, sequenceid=61, filesize=30.2 K 2024-12-08T21:14:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/c4e001cebd884f3f9f9b9c19fb698303 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c4e001cebd884f3f9f9b9c19fb698303 2024-12-08T21:14:44,514 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c4e001cebd884f3f9f9b9c19fb698303, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T21:14:44,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5d0b95429cf74e67bba23f984f00e06e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5d0b95429cf74e67bba23f984f00e06e 2024-12-08T21:14:44,521 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5d0b95429cf74e67bba23f984f00e06e, entries=150, sequenceid=61, filesize=11.7 K 2024-12-08T21:14:44,522 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cddc5d594981d66d83e5fc7c39ab45c4 in 1444ms, sequenceid=61, compaction requested=true 2024-12-08T21:14:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-08T21:14:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-08T21:14:44,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-08T21:14:44,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6020 sec 2024-12-08T21:14:44,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6080 sec 2024-12-08T21:14:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-08T21:14:45,068 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:45,069 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-08T21:14:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T21:14:45,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:45,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:45,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T21:14:45,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-08T21:14:45,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:45,223 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T21:14:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:45,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:45,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208893ce80d98b04323a66a907ee371a7c1_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692483260/Put/seqid=0 2024-12-08T21:14:45,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741974_1150 (size=12154) 2024-12-08T21:14:45,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:45,240 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208893ce80d98b04323a66a907ee371a7c1_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208893ce80d98b04323a66a907ee371a7c1_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/11b7610b910c4e14a0d21bcd22a39093, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:45,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/11b7610b910c4e14a0d21bcd22a39093 is 175, key is test_row_0/A:col10/1733692483260/Put/seqid=0 2024-12-08T21:14:45,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741975_1151 (size=30955) 2024-12-08T21:14:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T21:14:45,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:45,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692545448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692545449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692545450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692545458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692545458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,461 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,461 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692545558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692545568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692545568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,655 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/11b7610b910c4e14a0d21bcd22a39093 2024-12-08T21:14:45,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/a33c4d13800d41f1ac3e586886a566b3 is 50, key is test_row_0/B:col10/1733692483260/Put/seqid=0 2024-12-08T21:14:45,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741976_1152 (size=12001) 2024-12-08T21:14:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T21:14:45,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692545768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692545778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692545778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,067 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/a33c4d13800d41f1ac3e586886a566b3 2024-12-08T21:14:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/cca401ec00e5402eba85033bfe189f01 is 50, key is test_row_0/C:col10/1733692483260/Put/seqid=0 2024-12-08T21:14:46,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692546077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692546088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692546089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741977_1153 (size=12001) 2024-12-08T21:14:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T21:14:46,499 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/cca401ec00e5402eba85033bfe189f01 2024-12-08T21:14:46,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/11b7610b910c4e14a0d21bcd22a39093 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/11b7610b910c4e14a0d21bcd22a39093 2024-12-08T21:14:46,509 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/11b7610b910c4e14a0d21bcd22a39093, entries=150, sequenceid=98, filesize=30.2 K 2024-12-08T21:14:46,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/a33c4d13800d41f1ac3e586886a566b3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a33c4d13800d41f1ac3e586886a566b3 2024-12-08T21:14:46,514 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a33c4d13800d41f1ac3e586886a566b3, entries=150, sequenceid=98, filesize=11.7 K 2024-12-08T21:14:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/cca401ec00e5402eba85033bfe189f01 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/cca401ec00e5402eba85033bfe189f01 2024-12-08T21:14:46,522 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/cca401ec00e5402eba85033bfe189f01, entries=150, sequenceid=98, filesize=11.7 K 2024-12-08T21:14:46,523 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cddc5d594981d66d83e5fc7c39ab45c4 in 1299ms, sequenceid=98, compaction requested=true 2024-12-08T21:14:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:46,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-08T21:14:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-08T21:14:46,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-08T21:14:46,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4530 sec 2024-12-08T21:14:46,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4570 sec 2024-12-08T21:14:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:46,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T21:14:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:46,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208675d18ed779844768bc555b866e8c86f_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692486589/Put/seqid=0 2024-12-08T21:14:46,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741978_1154 (size=14594) 2024-12-08T21:14:46,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692486627, max timestamp=1733692486631Num uniques 859; 2024-12-08T21:14:46,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692486627, max timestamp=1733692486631Num uniques 859; 2024-12-08T21:14:46,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692486627, max timestamp=1733692486631Num uniques 859; 2024-12-08T21:14:46,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:46,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:46,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:14:46,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692546656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692546661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692546661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692546767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692546768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692546768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692546978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692546978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:46,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692546988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:46,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,007 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:47,011 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208675d18ed779844768bc555b866e8c86f_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208675d18ed779844768bc555b866e8c86f_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:47,013 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1429c61f9a51454db3db1540196f719a, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:47,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1429c61f9a51454db3db1540196f719a is 175, key is test_row_0/A:col10/1733692486589/Put/seqid=0 2024-12-08T21:14:47,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741979_1155 (size=39549) 2024-12-08T21:14:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-08T21:14:47,218 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:47,218 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-08T21:14:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T21:14:47,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:47,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:47,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692547288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692547288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692547298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T21:14:47,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T21:14:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1429c61f9a51454db3db1540196f719a 2024-12-08T21:14:47,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/aef208f8a144487dbb22fbaddf2b0a44 is 50, key is test_row_0/B:col10/1733692486589/Put/seqid=0 2024-12-08T21:14:47,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741980_1156 (size=12001) 2024-12-08T21:14:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692547478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692547478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,480 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,481 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T21:14:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:47,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T21:14:47,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T21:14:47,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:47,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692547798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692547798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692547808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:47,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T21:14:47,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/aef208f8a144487dbb22fbaddf2b0a44 2024-12-08T21:14:47,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:47,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/76c3686a87df4c8693a5f7ff7dea72bc is 50, key is test_row_0/C:col10/1733692486589/Put/seqid=0 2024-12-08T21:14:47,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741981_1157 (size=12001) 2024-12-08T21:14:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T21:14:47,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T21:14:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:47,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:48,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T21:14:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:48,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:48,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/76c3686a87df4c8693a5f7ff7dea72bc 2024-12-08T21:14:48,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1429c61f9a51454db3db1540196f719a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1429c61f9a51454db3db1540196f719a 2024-12-08T21:14:48,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1429c61f9a51454db3db1540196f719a, entries=200, sequenceid=109, filesize=38.6 K 2024-12-08T21:14:48,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/aef208f8a144487dbb22fbaddf2b0a44 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/aef208f8a144487dbb22fbaddf2b0a44 2024-12-08T21:14:48,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/aef208f8a144487dbb22fbaddf2b0a44, entries=150, sequenceid=109, filesize=11.7 K 2024-12-08T21:14:48,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/76c3686a87df4c8693a5f7ff7dea72bc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/76c3686a87df4c8693a5f7ff7dea72bc 2024-12-08T21:14:48,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/76c3686a87df4c8693a5f7ff7dea72bc, entries=150, sequenceid=109, filesize=11.7 K 2024-12-08T21:14:48,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cddc5d594981d66d83e5fc7c39ab45c4 in 1673ms, sequenceid=109, compaction requested=true 2024-12-08T21:14:48,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:48,264 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T21:14:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:48,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:48,264 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T21:14:48,265 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163369 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T21:14:48,265 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T21:14:48,265 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/B is initiating minor compaction (all files) 2024-12-08T21:14:48,265 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/A is initiating minor compaction (all files) 2024-12-08T21:14:48,265 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/A in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:48,265 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/B in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:48,266 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e0f9bdc9acc548c79a044d12d3644a1e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/4e08eb2502ca44e6a214ae120e94e145, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a2eec19589414799bac341e2322f1857, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/11b7610b910c4e14a0d21bcd22a39093, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1429c61f9a51454db3db1540196f719a] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=159.5 K 2024-12-08T21:14:48,266 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/e362f557e7be40b1952aba5729837260, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8f0aaa426d8d4006886c7faf1719982c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c4e001cebd884f3f9f9b9c19fb698303, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a33c4d13800d41f1ac3e586886a566b3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/aef208f8a144487dbb22fbaddf2b0a44] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=58.6 K 2024-12-08T21:14:48,266 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:48,266 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e0f9bdc9acc548c79a044d12d3644a1e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/4e08eb2502ca44e6a214ae120e94e145, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a2eec19589414799bac341e2322f1857, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/11b7610b910c4e14a0d21bcd22a39093, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1429c61f9a51454db3db1540196f719a] 2024-12-08T21:14:48,266 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e362f557e7be40b1952aba5729837260, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733692480785 2024-12-08T21:14:48,267 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0f9bdc9acc548c79a044d12d3644a1e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733692480785 2024-12-08T21:14:48,267 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f0aaa426d8d4006886c7faf1719982c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733692480912 2024-12-08T21:14:48,267 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e08eb2502ca44e6a214ae120e94e145, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733692480912 2024-12-08T21:14:48,267 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e001cebd884f3f9f9b9c19fb698303, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733692482073 2024-12-08T21:14:48,267 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2eec19589414799bac341e2322f1857, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733692482073 2024-12-08T21:14:48,267 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a33c4d13800d41f1ac3e586886a566b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733692483260 2024-12-08T21:14:48,267 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b7610b910c4e14a0d21bcd22a39093, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733692483260 2024-12-08T21:14:48,268 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting aef208f8a144487dbb22fbaddf2b0a44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733692485444 2024-12-08T21:14:48,268 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1429c61f9a51454db3db1540196f719a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733692485444 2024-12-08T21:14:48,274 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:48,277 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#B#compaction#137 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:48,277 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/a2e9708aaef54f458d719e757c0b5892 is 50, key is test_row_0/B:col10/1733692486589/Put/seqid=0 2024-12-08T21:14:48,281 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208940c536beaf6450b807575d5c6c1ae8d_cddc5d594981d66d83e5fc7c39ab45c4 store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:48,286 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208940c536beaf6450b807575d5c6c1ae8d_cddc5d594981d66d83e5fc7c39ab45c4, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:48,286 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208940c536beaf6450b807575d5c6c1ae8d_cddc5d594981d66d83e5fc7c39ab45c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:48,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-08T21:14:48,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:48,290 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T21:14:48,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:48,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:48,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:48,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741982_1158 (size=12173) 2024-12-08T21:14:48,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741983_1159 (size=4469) 2024-12-08T21:14:48,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d709cf52792d45d081b147d8495e29b3_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692486647/Put/seqid=0 2024-12-08T21:14:48,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741984_1160 (size=12304) 2024-12-08T21:14:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T21:14:48,603 INFO [master/e653e208f502:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-08T21:14:48,603 INFO [master/e653e208f502:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-08T21:14:48,699 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/a2e9708aaef54f458d719e757c0b5892 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a2e9708aaef54f458d719e757c0b5892 2024-12-08T21:14:48,703 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#A#compaction#138 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:48,704 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/07a6dfdf397f4a7e842f593c554c7c36 is 175, key is test_row_0/A:col10/1733692486589/Put/seqid=0 2024-12-08T21:14:48,709 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/B of cddc5d594981d66d83e5fc7c39ab45c4 into a2e9708aaef54f458d719e757c0b5892(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:48,709 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:48,709 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/B, priority=11, startTime=1733692488264; duration=0sec 2024-12-08T21:14:48,709 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:48,709 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:B 2024-12-08T21:14:48,709 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-08T21:14:48,710 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-08T21:14:48,711 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/C is initiating minor compaction (all files) 2024-12-08T21:14:48,711 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/C in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:48,711 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/dfa7f77d079349d69dcf1be962d79693, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5622d57b10704d4d8e0f822c2e67a256, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5d0b95429cf74e67bba23f984f00e06e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/cca401ec00e5402eba85033bfe189f01, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/76c3686a87df4c8693a5f7ff7dea72bc] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=58.6 K 2024-12-08T21:14:48,711 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting dfa7f77d079349d69dcf1be962d79693, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733692480785 2024-12-08T21:14:48,712 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5622d57b10704d4d8e0f822c2e67a256, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733692480912 2024-12-08T21:14:48,712 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d0b95429cf74e67bba23f984f00e06e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733692482073 2024-12-08T21:14:48,713 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting cca401ec00e5402eba85033bfe189f01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733692483260 2024-12-08T21:14:48,713 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c3686a87df4c8693a5f7ff7dea72bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733692485444 2024-12-08T21:14:48,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741985_1161 (size=31127) 2024-12-08T21:14:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:48,723 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:48,724 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5c444a729ee0434f9dbd086ca686017d is 50, key is test_row_0/C:col10/1733692486589/Put/seqid=0 2024-12-08T21:14:48,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d709cf52792d45d081b147d8495e29b3_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d709cf52792d45d081b147d8495e29b3_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3b1ad5ebee634a658e4628c81902f62e, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:48,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3b1ad5ebee634a658e4628c81902f62e is 175, key is test_row_0/A:col10/1733692486647/Put/seqid=0 2024-12-08T21:14:48,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741986_1162 (size=12173) 2024-12-08T21:14:48,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741987_1163 (size=31105) 2024-12-08T21:14:48,731 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3b1ad5ebee634a658e4628c81902f62e 2024-12-08T21:14:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/8714ed7378a14d2c87d968f1b2cdfe8c is 50, key is test_row_0/B:col10/1733692486647/Put/seqid=0 2024-12-08T21:14:48,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741988_1164 (size=12151) 2024-12-08T21:14:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:48,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:48,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692548835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:48,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692548835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:48,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692548836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:48,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692548948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692548948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692548948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:48,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:49,119 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/07a6dfdf397f4a7e842f593c554c7c36 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/07a6dfdf397f4a7e842f593c554c7c36 2024-12-08T21:14:49,125 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/A of cddc5d594981d66d83e5fc7c39ab45c4 into 07a6dfdf397f4a7e842f593c554c7c36(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:49,125 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:49,125 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/A, priority=11, startTime=1733692488264; duration=0sec 2024-12-08T21:14:49,125 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:49,125 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:A 2024-12-08T21:14:49,134 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/5c444a729ee0434f9dbd086ca686017d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5c444a729ee0434f9dbd086ca686017d 2024-12-08T21:14:49,141 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/C of cddc5d594981d66d83e5fc7c39ab45c4 into 5c444a729ee0434f9dbd086ca686017d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:49,141 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:49,141 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/C, priority=11, startTime=1733692488264; duration=0sec 2024-12-08T21:14:49,141 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:49,141 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:C 2024-12-08T21:14:49,144 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/8714ed7378a14d2c87d968f1b2cdfe8c 2024-12-08T21:14:49,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6dc317ec7b3f4d81b5f4eb804d52fd95 is 50, key is test_row_0/C:col10/1733692486647/Put/seqid=0 2024-12-08T21:14:49,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692549158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692549158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692549158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:49,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741989_1165 (size=12151) 2024-12-08T21:14:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T21:14:49,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692549467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:49,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:49,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692549468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:49,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692549468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:49,568 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6dc317ec7b3f4d81b5f4eb804d52fd95 2024-12-08T21:14:49,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3b1ad5ebee634a658e4628c81902f62e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3b1ad5ebee634a658e4628c81902f62e 2024-12-08T21:14:49,578 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3b1ad5ebee634a658e4628c81902f62e, entries=150, sequenceid=146, filesize=30.4 K 2024-12-08T21:14:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/8714ed7378a14d2c87d968f1b2cdfe8c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8714ed7378a14d2c87d968f1b2cdfe8c 2024-12-08T21:14:49,583 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8714ed7378a14d2c87d968f1b2cdfe8c, entries=150, sequenceid=146, filesize=11.9 K 2024-12-08T21:14:49,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6dc317ec7b3f4d81b5f4eb804d52fd95 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6dc317ec7b3f4d81b5f4eb804d52fd95 2024-12-08T21:14:49,589 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6dc317ec7b3f4d81b5f4eb804d52fd95, entries=150, sequenceid=146, filesize=11.9 K 2024-12-08T21:14:49,590 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=60.38 KB/61830 for cddc5d594981d66d83e5fc7c39ab45c4 in 1300ms, sequenceid=146, compaction requested=false 2024-12-08T21:14:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:49,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-08T21:14:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-08T21:14:49,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-08T21:14:49,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3710 sec 2024-12-08T21:14:49,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.3770 sec 2024-12-08T21:14:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:49,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:14:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:49,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:49,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085ad7705e28c34e6a9b594a528a19bafb_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692488834/Put/seqid=0 2024-12-08T21:14:49,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741990_1166 (size=12304) 2024-12-08T21:14:49,997 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:50,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085ad7705e28c34e6a9b594a528a19bafb_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085ad7705e28c34e6a9b594a528a19bafb_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:50,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c4ede7df97214255b4c9aad2287bb9ae, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:50,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c4ede7df97214255b4c9aad2287bb9ae is 175, key is test_row_0/A:col10/1733692488834/Put/seqid=0 2024-12-08T21:14:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692550007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692550008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692550009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741991_1167 (size=31105) 2024-12-08T21:14:50,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692550118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692550118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692550118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692550328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692550328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692550328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,423 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c4ede7df97214255b4c9aad2287bb9ae 2024-12-08T21:14:50,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/c29c192b18cd4550976ea195d5abf3be is 50, key is test_row_0/B:col10/1733692488834/Put/seqid=0 2024-12-08T21:14:50,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741992_1168 (size=12151) 2024-12-08T21:14:50,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692550637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692550638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:50,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692550638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:50,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:50,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/c29c192b18cd4550976ea195d5abf3be 2024-12-08T21:14:50,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/2ec1d571e84946ddb8612cf559d29ddf is 50, key is test_row_0/C:col10/1733692488834/Put/seqid=0 2024-12-08T21:14:50,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741993_1169 (size=12151) 2024-12-08T21:14:51,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692551148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692551148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692551158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/2ec1d571e84946ddb8612cf559d29ddf 2024-12-08T21:14:51,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c4ede7df97214255b4c9aad2287bb9ae as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c4ede7df97214255b4c9aad2287bb9ae 2024-12-08T21:14:51,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c4ede7df97214255b4c9aad2287bb9ae, entries=150, sequenceid=162, filesize=30.4 K 2024-12-08T21:14:51,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/c29c192b18cd4550976ea195d5abf3be as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c29c192b18cd4550976ea195d5abf3be 2024-12-08T21:14:51,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c29c192b18cd4550976ea195d5abf3be, entries=150, sequenceid=162, filesize=11.9 K 2024-12-08T21:14:51,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/2ec1d571e84946ddb8612cf559d29ddf as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2ec1d571e84946ddb8612cf559d29ddf 2024-12-08T21:14:51,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2ec1d571e84946ddb8612cf559d29ddf, entries=150, sequenceid=162, filesize=11.9 K 2024-12-08T21:14:51,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cddc5d594981d66d83e5fc7c39ab45c4 in 1298ms, sequenceid=162, compaction requested=true 2024-12-08T21:14:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:51,277 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:51,277 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:51,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:51,278 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93337 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:51,278 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:51,278 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/B is initiating minor compaction (all files) 2024-12-08T21:14:51,278 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/A is initiating minor compaction (all files) 2024-12-08T21:14:51,278 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/A in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,278 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/B in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,279 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/07a6dfdf397f4a7e842f593c554c7c36, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3b1ad5ebee634a658e4628c81902f62e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c4ede7df97214255b4c9aad2287bb9ae] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=91.1 K 2024-12-08T21:14:51,279 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a2e9708aaef54f458d719e757c0b5892, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8714ed7378a14d2c87d968f1b2cdfe8c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c29c192b18cd4550976ea195d5abf3be] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=35.6 K 2024-12-08T21:14:51,279 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,279 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/07a6dfdf397f4a7e842f593c554c7c36, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3b1ad5ebee634a658e4628c81902f62e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c4ede7df97214255b4c9aad2287bb9ae] 2024-12-08T21:14:51,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e9708aaef54f458d719e757c0b5892, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733692485444 2024-12-08T21:14:51,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8714ed7378a14d2c87d968f1b2cdfe8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733692486647 2024-12-08T21:14:51,279 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a6dfdf397f4a7e842f593c554c7c36, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733692485444 2024-12-08T21:14:51,280 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c29c192b18cd4550976ea195d5abf3be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733692488833 2024-12-08T21:14:51,280 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b1ad5ebee634a658e4628c81902f62e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733692486647 2024-12-08T21:14:51,280 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ede7df97214255b4c9aad2287bb9ae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733692488833 2024-12-08T21:14:51,285 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:51,289 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412082fbb31f34f3c458f848bb76f98ae67c0_cddc5d594981d66d83e5fc7c39ab45c4 store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:51,289 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:51,290 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/bf4392816ef44e6d997b384200a38f51 is 50, key is test_row_0/B:col10/1733692488834/Put/seqid=0 2024-12-08T21:14:51,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412082fbb31f34f3c458f848bb76f98ae67c0_cddc5d594981d66d83e5fc7c39ab45c4, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:51,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082fbb31f34f3c458f848bb76f98ae67c0_cddc5d594981d66d83e5fc7c39ab45c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:51,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741994_1170 (size=12425) 2024-12-08T21:14:51,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741995_1171 (size=4469) 2024-12-08T21:14:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-08T21:14:51,378 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:51,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-08T21:14:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T21:14:51,380 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:51,381 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:51,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T21:14:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:51,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:14:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:51,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:51,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208be5803f3a5af407fbe2105d1bb9def05_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692490006/Put/seqid=0 2024-12-08T21:14:51,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741996_1172 (size=14794) 2024-12-08T21:14:51,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692551520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692551521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:51,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:51,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692551628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692551628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T21:14:51,699 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/bf4392816ef44e6d997b384200a38f51 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/bf4392816ef44e6d997b384200a38f51 2024-12-08T21:14:51,704 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/B of cddc5d594981d66d83e5fc7c39ab45c4 into bf4392816ef44e6d997b384200a38f51(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:51,704 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:51,704 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/B, priority=13, startTime=1733692491277; duration=0sec 2024-12-08T21:14:51,704 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:51,704 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:B 2024-12-08T21:14:51,704 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:14:51,704 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#A#compaction#146 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:51,705 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/f34d9e02a04d432f94f8591386256e15 is 175, key is test_row_0/A:col10/1733692488834/Put/seqid=0 2024-12-08T21:14:51,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36475 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:14:51,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/C is initiating minor compaction (all files) 2024-12-08T21:14:51,707 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/C in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,707 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5c444a729ee0434f9dbd086ca686017d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6dc317ec7b3f4d81b5f4eb804d52fd95, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2ec1d571e84946ddb8612cf559d29ddf] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=35.6 K 2024-12-08T21:14:51,707 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c444a729ee0434f9dbd086ca686017d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733692485444 2024-12-08T21:14:51,708 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc317ec7b3f4d81b5f4eb804d52fd95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733692486647 2024-12-08T21:14:51,708 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec1d571e84946ddb8612cf559d29ddf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733692488833 2024-12-08T21:14:51,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741997_1173 (size=31379) 2024-12-08T21:14:51,717 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#C#compaction#149 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:51,718 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/0d9fccbcec2f4632b3ff70b68fbd6241 is 50, key is test_row_0/C:col10/1733692488834/Put/seqid=0 2024-12-08T21:14:51,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741998_1174 (size=12425) 2024-12-08T21:14:51,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692551838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692551838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:51,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:51,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:51,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:51,916 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208be5803f3a5af407fbe2105d1bb9def05_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208be5803f3a5af407fbe2105d1bb9def05_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:51,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/6841c2996c034f82a9c3424cd4d79840, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:51,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/6841c2996c034f82a9c3424cd4d79840 is 175, key is test_row_0/A:col10/1733692490006/Put/seqid=0 2024-12-08T21:14:51,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741999_1175 (size=39749) 2024-12-08T21:14:51,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:51,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T21:14:52,114 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/f34d9e02a04d432f94f8591386256e15 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/f34d9e02a04d432f94f8591386256e15 2024-12-08T21:14:52,119 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/A of cddc5d594981d66d83e5fc7c39ab45c4 into f34d9e02a04d432f94f8591386256e15(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:52,119 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:52,119 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/A, priority=13, startTime=1733692491277; duration=0sec 2024-12-08T21:14:52,119 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:52,119 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:A 2024-12-08T21:14:52,126 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/0d9fccbcec2f4632b3ff70b68fbd6241 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d9fccbcec2f4632b3ff70b68fbd6241 2024-12-08T21:14:52,130 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/C of cddc5d594981d66d83e5fc7c39ab45c4 into 0d9fccbcec2f4632b3ff70b68fbd6241(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:52,130 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:52,130 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/C, priority=13, startTime=1733692491277; duration=0sec 2024-12-08T21:14:52,131 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:52,131 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:C 2024-12-08T21:14:52,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692552147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:52,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692552148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:52,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692552158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:52,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692552158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:52,161 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:52,161 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:52,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692552178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:52,181 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:52,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,327 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/6841c2996c034f82a9c3424cd4d79840 2024-12-08T21:14:52,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/ce3835fb763b47a4acbc4211a7106578 is 50, key is test_row_0/B:col10/1733692490006/Put/seqid=0 2024-12-08T21:14:52,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742000_1176 (size=12151) 2024-12-08T21:14:52,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T21:14:52,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:52,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:52,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692552658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:52,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692552658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:52,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/ce3835fb763b47a4acbc4211a7106578 2024-12-08T21:14:52,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6759bad476324ed2bc62109b9be9600d is 50, key is test_row_0/C:col10/1733692490006/Put/seqid=0 2024-12-08T21:14:52,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:52,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742001_1177 (size=12151) 2024-12-08T21:14:52,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6759bad476324ed2bc62109b9be9600d 2024-12-08T21:14:52,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/6841c2996c034f82a9c3424cd4d79840 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/6841c2996c034f82a9c3424cd4d79840 2024-12-08T21:14:52,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/6841c2996c034f82a9c3424cd4d79840, entries=200, sequenceid=182, filesize=38.8 K 2024-12-08T21:14:52,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/ce3835fb763b47a4acbc4211a7106578 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ce3835fb763b47a4acbc4211a7106578 2024-12-08T21:14:52,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ce3835fb763b47a4acbc4211a7106578, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T21:14:52,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6759bad476324ed2bc62109b9be9600d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6759bad476324ed2bc62109b9be9600d 2024-12-08T21:14:52,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6759bad476324ed2bc62109b9be9600d, entries=150, sequenceid=182, filesize=11.9 K 2024-12-08T21:14:52,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cddc5d594981d66d83e5fc7c39ab45c4 in 1291ms, sequenceid=182, compaction requested=false 2024-12-08T21:14:52,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:52,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-08T21:14:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:52,909 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:14:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:52,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:52,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d5ab0f9d110246df98c2c4e0c86ed572_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692491518/Put/seqid=0 2024-12-08T21:14:52,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742002_1178 (size=12304) 2024-12-08T21:14:52,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:52,935 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208d5ab0f9d110246df98c2c4e0c86ed572_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d5ab0f9d110246df98c2c4e0c86ed572_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:52,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2d13cd9d436f4fd2a8988edac6632dde, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:52,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2d13cd9d436f4fd2a8988edac6632dde is 175, key is test_row_0/A:col10/1733692491518/Put/seqid=0 2024-12-08T21:14:52,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742003_1179 (size=31105) 2024-12-08T21:14:52,955 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2d13cd9d436f4fd2a8988edac6632dde 2024-12-08T21:14:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/108571a3393043ad9c5a6891093eefc8 is 50, key is test_row_0/B:col10/1733692491518/Put/seqid=0 2024-12-08T21:14:52,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742004_1180 (size=12151) 2024-12-08T21:14:53,369 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/108571a3393043ad9c5a6891093eefc8 2024-12-08T21:14:53,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6ed5b757f2d643a788d537ee237ae296 is 50, key is test_row_0/C:col10/1733692491518/Put/seqid=0 2024-12-08T21:14:53,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742005_1181 (size=12151) 2024-12-08T21:14:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T21:14:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:53,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:53,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692553694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692553695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:53,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:53,786 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6ed5b757f2d643a788d537ee237ae296 2024-12-08T21:14:53,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2d13cd9d436f4fd2a8988edac6632dde as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2d13cd9d436f4fd2a8988edac6632dde 2024-12-08T21:14:53,794 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2d13cd9d436f4fd2a8988edac6632dde, entries=150, sequenceid=197, filesize=30.4 K 2024-12-08T21:14:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/108571a3393043ad9c5a6891093eefc8 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/108571a3393043ad9c5a6891093eefc8 2024-12-08T21:14:53,800 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/108571a3393043ad9c5a6891093eefc8, entries=150, sequenceid=197, filesize=11.9 K 2024-12-08T21:14:53,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6ed5b757f2d643a788d537ee237ae296 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6ed5b757f2d643a788d537ee237ae296 2024-12-08T21:14:53,805 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6ed5b757f2d643a788d537ee237ae296, entries=150, sequenceid=197, filesize=11.9 K 2024-12-08T21:14:53,806 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cddc5d594981d66d83e5fc7c39ab45c4 in 897ms, sequenceid=197, compaction requested=true 2024-12-08T21:14:53,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:53,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:53,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-08T21:14:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-08T21:14:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:53,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T21:14:53,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:53,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:53,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:53,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:53,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:53,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:14:53,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-08T21:14:53,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4270 sec 2024-12-08T21:14:53,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4780 sec 2024-12-08T21:14:53,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208879c524b42744645b82b6f5be4bb6a44_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692493693/Put/seqid=0 2024-12-08T21:14:53,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692553875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:53,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:53,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692553881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:53,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742006_1182 (size=12304) 2024-12-08T21:14:53,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:53,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208879c524b42744645b82b6f5be4bb6a44_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208879c524b42744645b82b6f5be4bb6a44_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:53,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c7201d6eb1d44b95930b74dc0c04e815, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:53,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c7201d6eb1d44b95930b74dc0c04e815 is 175, key is test_row_0/A:col10/1733692493693/Put/seqid=0 2024-12-08T21:14:53,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742007_1183 (size=31105) 2024-12-08T21:14:53,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c7201d6eb1d44b95930b74dc0c04e815 2024-12-08T21:14:53,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/549ce694c3f94902a556a3db4607478e is 50, key is test_row_0/B:col10/1733692493693/Put/seqid=0 2024-12-08T21:14:53,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742008_1184 (size=12151) 2024-12-08T21:14:53,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692553987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692553987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692554168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:54,170 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692554188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:54,192 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692554197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692554197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692554208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:54,208 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/549ce694c3f94902a556a3db4607478e 2024-12-08T21:14:54,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/82b195f759f145b2be149c7cd404ba6f is 50, key is test_row_0/C:col10/1733692493693/Put/seqid=0 2024-12-08T21:14:54,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742009_1185 (size=12151) 2024-12-08T21:14:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692554508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:54,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692554508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:54,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/82b195f759f145b2be149c7cd404ba6f 2024-12-08T21:14:54,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/c7201d6eb1d44b95930b74dc0c04e815 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c7201d6eb1d44b95930b74dc0c04e815 2024-12-08T21:14:54,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c7201d6eb1d44b95930b74dc0c04e815, entries=150, sequenceid=218, filesize=30.4 K 2024-12-08T21:14:54,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/549ce694c3f94902a556a3db4607478e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/549ce694c3f94902a556a3db4607478e 2024-12-08T21:14:54,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/549ce694c3f94902a556a3db4607478e, entries=150, sequenceid=218, filesize=11.9 K 2024-12-08T21:14:54,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/82b195f759f145b2be149c7cd404ba6f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/82b195f759f145b2be149c7cd404ba6f 2024-12-08T21:14:54,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/82b195f759f145b2be149c7cd404ba6f, entries=150, sequenceid=218, filesize=11.9 K 2024-12-08T21:14:54,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for cddc5d594981d66d83e5fc7c39ab45c4 in 980ms, sequenceid=218, compaction requested=true 2024-12-08T21:14:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:54,789 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:54,789 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:54,790 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133338 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:54,790 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:54,791 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/A is initiating minor compaction (all files) 2024-12-08T21:14:54,791 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/B is initiating minor compaction (all files) 2024-12-08T21:14:54,791 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/A in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:54,791 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/B in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:54,791 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/bf4392816ef44e6d997b384200a38f51, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ce3835fb763b47a4acbc4211a7106578, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/108571a3393043ad9c5a6891093eefc8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/549ce694c3f94902a556a3db4607478e] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=47.7 K 2024-12-08T21:14:54,791 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/f34d9e02a04d432f94f8591386256e15, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/6841c2996c034f82a9c3424cd4d79840, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2d13cd9d436f4fd2a8988edac6632dde, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c7201d6eb1d44b95930b74dc0c04e815] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=130.2 K 2024-12-08T21:14:54,791 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:54,791 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/f34d9e02a04d432f94f8591386256e15, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/6841c2996c034f82a9c3424cd4d79840, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2d13cd9d436f4fd2a8988edac6632dde, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c7201d6eb1d44b95930b74dc0c04e815] 2024-12-08T21:14:54,791 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4392816ef44e6d997b384200a38f51, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733692488833 2024-12-08T21:14:54,791 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting f34d9e02a04d432f94f8591386256e15, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733692488833 2024-12-08T21:14:54,792 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3835fb763b47a4acbc4211a7106578, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733692490006 2024-12-08T21:14:54,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6841c2996c034f82a9c3424cd4d79840, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733692490000 2024-12-08T21:14:54,792 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 108571a3393043ad9c5a6891093eefc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733692491516 2024-12-08T21:14:54,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d13cd9d436f4fd2a8988edac6632dde, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733692491516 2024-12-08T21:14:54,792 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 549ce694c3f94902a556a3db4607478e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733692493693 2024-12-08T21:14:54,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7201d6eb1d44b95930b74dc0c04e815, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733692493693 2024-12-08T21:14:54,807 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:54,810 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:54,811 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/459b149c2c2440d587ffecc09db6d5c4 is 50, key is test_row_0/B:col10/1733692493693/Put/seqid=0 2024-12-08T21:14:54,817 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208fe7930f11da444a4856288a0c9feae00_cddc5d594981d66d83e5fc7c39ab45c4 store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:54,821 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208fe7930f11da444a4856288a0c9feae00_cddc5d594981d66d83e5fc7c39ab45c4, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:54,821 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe7930f11da444a4856288a0c9feae00_cddc5d594981d66d83e5fc7c39ab45c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:54,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742011_1187 (size=4469) 2024-12-08T21:14:54,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742010_1186 (size=12561) 2024-12-08T21:14:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:55,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:14:55,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:55,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:55,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c657ecb3e238467995ed74f49f16ab6a_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692493871/Put/seqid=0 2024-12-08T21:14:55,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742012_1188 (size=14794) 2024-12-08T21:14:55,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692555056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692555061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692555168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692555168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,227 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#A#compaction#158 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:55,228 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/a8e2fad7ea904befbbef5816aabe2f3a is 175, key is test_row_0/A:col10/1733692493693/Put/seqid=0 2024-12-08T21:14:55,231 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/459b149c2c2440d587ffecc09db6d5c4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/459b149c2c2440d587ffecc09db6d5c4 2024-12-08T21:14:55,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742013_1189 (size=31515) 2024-12-08T21:14:55,237 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/B of cddc5d594981d66d83e5fc7c39ab45c4 into 459b149c2c2440d587ffecc09db6d5c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:55,237 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:55,238 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/B, priority=12, startTime=1733692494789; duration=0sec 2024-12-08T21:14:55,238 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:55,238 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:B 2024-12-08T21:14:55,238 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:55,239 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:55,239 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/C is initiating minor compaction (all files) 2024-12-08T21:14:55,239 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/C in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:55,240 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d9fccbcec2f4632b3ff70b68fbd6241, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6759bad476324ed2bc62109b9be9600d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6ed5b757f2d643a788d537ee237ae296, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/82b195f759f145b2be149c7cd404ba6f] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=47.7 K 2024-12-08T21:14:55,240 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d9fccbcec2f4632b3ff70b68fbd6241, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1733692488833 2024-12-08T21:14:55,240 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6759bad476324ed2bc62109b9be9600d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733692490006 2024-12-08T21:14:55,240 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ed5b757f2d643a788d537ee237ae296, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733692491516 2024-12-08T21:14:55,241 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 82b195f759f145b2be149c7cd404ba6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733692493693 2024-12-08T21:14:55,247 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:55,247 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/1e52b8ae16b6432fa4795ab3309d5e18 is 50, key is test_row_0/C:col10/1733692493693/Put/seqid=0 2024-12-08T21:14:55,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742014_1190 (size=12561) 2024-12-08T21:14:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692555378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692555378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:55,436 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c657ecb3e238467995ed74f49f16ab6a_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c657ecb3e238467995ed74f49f16ab6a_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:55,437 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/380a42aeae724b739af875a2a51ddb21, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:55,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/380a42aeae724b739af875a2a51ddb21 is 175, key is test_row_0/A:col10/1733692493871/Put/seqid=0 2024-12-08T21:14:55,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742015_1191 (size=39749) 2024-12-08T21:14:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-08T21:14:55,548 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:55,548 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-08T21:14:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T21:14:55,550 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:55,553 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:55,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:55,637 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/a8e2fad7ea904befbbef5816aabe2f3a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a8e2fad7ea904befbbef5816aabe2f3a 2024-12-08T21:14:55,642 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/A of cddc5d594981d66d83e5fc7c39ab45c4 into a8e2fad7ea904befbbef5816aabe2f3a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:55,642 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:55,642 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/A, priority=12, startTime=1733692494789; duration=0sec 2024-12-08T21:14:55,642 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:55,642 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:A 2024-12-08T21:14:55,656 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/1e52b8ae16b6432fa4795ab3309d5e18 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/1e52b8ae16b6432fa4795ab3309d5e18 2024-12-08T21:14:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T21:14:55,661 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/C of cddc5d594981d66d83e5fc7c39ab45c4 into 1e52b8ae16b6432fa4795ab3309d5e18(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:55,661 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:55,661 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/C, priority=12, startTime=1733692494789; duration=0sec 2024-12-08T21:14:55,661 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:55,661 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:C 2024-12-08T21:14:55,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692555688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692555688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:55,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:55,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:55,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:55,842 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/380a42aeae724b739af875a2a51ddb21 2024-12-08T21:14:55,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/cc1020f69a764ca88e78700c7700b32b is 50, key is test_row_0/B:col10/1733692493871/Put/seqid=0 2024-12-08T21:14:55,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742016_1192 (size=12151) 2024-12-08T21:14:55,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:55,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:55,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T21:14:56,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:56,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:56,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:56,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T21:14:56,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:56,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692556198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692556198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:56,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:56,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/cc1020f69a764ca88e78700c7700b32b 2024-12-08T21:14:56,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/0d247707c91f40a2a9e9f3246a8e4bf5 is 50, key is test_row_0/C:col10/1733692493871/Put/seqid=0 2024-12-08T21:14:56,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742017_1193 (size=12151) 2024-12-08T21:14:56,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:56,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:56,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:14:56,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/0d247707c91f40a2a9e9f3246a8e4bf5 2024-12-08T21:14:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/380a42aeae724b739af875a2a51ddb21 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/380a42aeae724b739af875a2a51ddb21 2024-12-08T21:14:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T21:14:56,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/380a42aeae724b739af875a2a51ddb21, entries=200, sequenceid=230, filesize=38.8 K 2024-12-08T21:14:56,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/cc1020f69a764ca88e78700c7700b32b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/cc1020f69a764ca88e78700c7700b32b 2024-12-08T21:14:56,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/cc1020f69a764ca88e78700c7700b32b, entries=150, sequenceid=230, filesize=11.9 K 2024-12-08T21:14:56,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/0d247707c91f40a2a9e9f3246a8e4bf5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d247707c91f40a2a9e9f3246a8e4bf5 2024-12-08T21:14:56,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d247707c91f40a2a9e9f3246a8e4bf5, entries=150, sequenceid=230, filesize=11.9 K 2024-12-08T21:14:56,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cddc5d594981d66d83e5fc7c39ab45c4 in 1678ms, sequenceid=230, compaction requested=false 2024-12-08T21:14:56,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:56,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-08T21:14:56,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:56,774 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:14:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:56,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:56,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088456d1664b24496dabb72783297b6eea_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692495037/Put/seqid=0 2024-12-08T21:14:56,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742018_1194 (size=12304) 2024-12-08T21:14:57,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:57,193 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088456d1664b24496dabb72783297b6eea_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088456d1664b24496dabb72783297b6eea_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1e25bda5ce34414798d871f00d2d2dfe, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1e25bda5ce34414798d871f00d2d2dfe is 175, key is test_row_0/A:col10/1733692495037/Put/seqid=0 2024-12-08T21:14:57,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742019_1195 (size=31105) 2024-12-08T21:14:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:14:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692557221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:57,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692557224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:57,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692557327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:57,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692557338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:57,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692557537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:57,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692557548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:57,603 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1e25bda5ce34414798d871f00d2d2dfe 2024-12-08T21:14:57,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/3a683409302b488986518604a00dd5ee is 50, key is test_row_0/B:col10/1733692495037/Put/seqid=0 2024-12-08T21:14:57,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742020_1196 (size=12151) 2024-12-08T21:14:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T21:14:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692557848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:57,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692557858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:58,014 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/3a683409302b488986518604a00dd5ee 2024-12-08T21:14:58,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/9f947fbb854148d9aa4b768bbc461505 is 50, key is test_row_0/C:col10/1733692495037/Put/seqid=0 2024-12-08T21:14:58,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742021_1197 (size=12151) 2024-12-08T21:14:58,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56574 deadline: 1733692558198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:58,199 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:58,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56578 deadline: 1733692558228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:58,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56628 deadline: 1733692558228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:58,229 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:58,229 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in cddc5d594981d66d83e5fc7c39ab45c4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:14:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:58,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:58,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692558358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:58,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692558368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:58,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:58,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/9f947fbb854148d9aa4b768bbc461505 2024-12-08T21:14:58,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/1e25bda5ce34414798d871f00d2d2dfe as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1e25bda5ce34414798d871f00d2d2dfe 2024-12-08T21:14:58,434 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1e25bda5ce34414798d871f00d2d2dfe, entries=150, sequenceid=253, filesize=30.4 K 2024-12-08T21:14:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/3a683409302b488986518604a00dd5ee as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/3a683409302b488986518604a00dd5ee 2024-12-08T21:14:58,439 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/3a683409302b488986518604a00dd5ee, entries=150, sequenceid=253, filesize=11.9 K 2024-12-08T21:14:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/9f947fbb854148d9aa4b768bbc461505 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/9f947fbb854148d9aa4b768bbc461505 2024-12-08T21:14:58,444 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/9f947fbb854148d9aa4b768bbc461505, entries=150, sequenceid=253, filesize=11.9 K 2024-12-08T21:14:58,445 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cddc5d594981d66d83e5fc7c39ab45c4 in 1671ms, sequenceid=253, compaction requested=true 2024-12-08T21:14:58,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:58,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:58,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-08T21:14:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-08T21:14:58,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-08T21:14:58,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8930 sec 2024-12-08T21:14:58,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.9000 sec 2024-12-08T21:14:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:59,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:14:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:59,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:59,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120899595b5a668641c6afa8d6cf9a2d8f2c_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692497218/Put/seqid=0 2024-12-08T21:14:59,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742022_1198 (size=14994) 2024-12-08T21:14:59,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:14:59,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120899595b5a668641c6afa8d6cf9a2d8f2c_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120899595b5a668641c6afa8d6cf9a2d8f2c_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:14:59,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3c20d34fcc544d6e99a1f62a956c0047, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:59,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3c20d34fcc544d6e99a1f62a956c0047 is 175, key is test_row_0/A:col10/1733692497218/Put/seqid=0 2024-12-08T21:14:59,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742023_1199 (size=39949) 2024-12-08T21:14:59,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3c20d34fcc544d6e99a1f62a956c0047 2024-12-08T21:14:59,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/dc23cd7fb3394ce782e3c737c998b7c5 is 50, key is test_row_0/B:col10/1733692497218/Put/seqid=0 2024-12-08T21:14:59,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742024_1200 (size=12301) 2024-12-08T21:14:59,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/dc23cd7fb3394ce782e3c737c998b7c5 2024-12-08T21:14:59,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692559415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:59,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6a359cc1eb30433e97ab437c2c2da1fb is 50, key is test_row_0/C:col10/1733692497218/Put/seqid=0 2024-12-08T21:14:59,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692559418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:59,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742025_1201 (size=12301) 2024-12-08T21:14:59,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692559528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692559528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-08T21:14:59,718 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:14:59,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:14:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-08T21:14:59,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T21:14:59,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:14:59,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:14:59,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:14:59,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692559738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:59,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:14:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692559738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:14:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:14:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:14:59,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6a359cc1eb30433e97ab437c2c2da1fb 2024-12-08T21:14:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T21:14:59,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3c20d34fcc544d6e99a1f62a956c0047 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3c20d34fcc544d6e99a1f62a956c0047 2024-12-08T21:14:59,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3c20d34fcc544d6e99a1f62a956c0047, entries=200, sequenceid=266, filesize=39.0 K 2024-12-08T21:14:59,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/dc23cd7fb3394ce782e3c737c998b7c5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/dc23cd7fb3394ce782e3c737c998b7c5 2024-12-08T21:14:59,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/dc23cd7fb3394ce782e3c737c998b7c5, entries=150, sequenceid=266, filesize=12.0 K 2024-12-08T21:14:59,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6a359cc1eb30433e97ab437c2c2da1fb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6a359cc1eb30433e97ab437c2c2da1fb 2024-12-08T21:14:59,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6a359cc1eb30433e97ab437c2c2da1fb, entries=150, sequenceid=266, filesize=12.0 K 2024-12-08T21:14:59,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for cddc5d594981d66d83e5fc7c39ab45c4 in 473ms, sequenceid=266, compaction requested=true 2024-12-08T21:14:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:14:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:59,844 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:14:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:59,844 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:14:59,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:59,846 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49164 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:59,846 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142318 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:59,846 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/A is initiating minor compaction (all files) 2024-12-08T21:14:59,846 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/B is initiating minor compaction (all files) 2024-12-08T21:14:59,846 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/B in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:59,846 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/A in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:59,846 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/459b149c2c2440d587ffecc09db6d5c4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/cc1020f69a764ca88e78700c7700b32b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/3a683409302b488986518604a00dd5ee, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/dc23cd7fb3394ce782e3c737c998b7c5] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=48.0 K 2024-12-08T21:14:59,846 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a8e2fad7ea904befbbef5816aabe2f3a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/380a42aeae724b739af875a2a51ddb21, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1e25bda5ce34414798d871f00d2d2dfe, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3c20d34fcc544d6e99a1f62a956c0047] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=139.0 K 2024-12-08T21:14:59,846 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:59,846 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a8e2fad7ea904befbbef5816aabe2f3a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/380a42aeae724b739af875a2a51ddb21, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1e25bda5ce34414798d871f00d2d2dfe, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3c20d34fcc544d6e99a1f62a956c0047] 2024-12-08T21:14:59,846 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 459b149c2c2440d587ffecc09db6d5c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733692493693 2024-12-08T21:14:59,847 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting cc1020f69a764ca88e78700c7700b32b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733692493868 2024-12-08T21:14:59,847 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8e2fad7ea904befbbef5816aabe2f3a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733692493693 2024-12-08T21:14:59,847 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a683409302b488986518604a00dd5ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692495037 2024-12-08T21:14:59,847 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 380a42aeae724b739af875a2a51ddb21, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733692493868 2024-12-08T21:14:59,848 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting dc23cd7fb3394ce782e3c737c998b7c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733692497210 2024-12-08T21:14:59,848 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e25bda5ce34414798d871f00d2d2dfe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692495037 2024-12-08T21:14:59,848 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c20d34fcc544d6e99a1f62a956c0047, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733692497210 2024-12-08T21:14:59,855 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:59,857 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:59,858 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/f2c1cca3a10a4bcfbbf42af2b6de65cf is 50, key is test_row_0/B:col10/1733692497218/Put/seqid=0 2024-12-08T21:14:59,869 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208147c2f064ff64810bcc5a9075a0de075_cddc5d594981d66d83e5fc7c39ab45c4 store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:59,871 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208147c2f064ff64810bcc5a9075a0de075_cddc5d594981d66d83e5fc7c39ab45c4, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:59,872 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208147c2f064ff64810bcc5a9075a0de075_cddc5d594981d66d83e5fc7c39ab45c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:14:59,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742026_1202 (size=12847) 2024-12-08T21:14:59,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-08T21:14:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:59,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T21:14:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:14:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:14:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:14:59,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:14:59,880 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/f2c1cca3a10a4bcfbbf42af2b6de65cf as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/f2c1cca3a10a4bcfbbf42af2b6de65cf 2024-12-08T21:14:59,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080ccf3e8ff9ce41aead04fd4f37a56881_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692499414/Put/seqid=0 2024-12-08T21:14:59,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742027_1203 (size=4469) 2024-12-08T21:14:59,887 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/B of cddc5d594981d66d83e5fc7c39ab45c4 into f2c1cca3a10a4bcfbbf42af2b6de65cf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:59,887 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:59,887 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/B, priority=12, startTime=1733692499844; duration=0sec 2024-12-08T21:14:59,887 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:14:59,887 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:B 2024-12-08T21:14:59,887 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:14:59,887 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#A#compaction#170 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:59,888 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/655f6817e7ca43fe9f39934c558c6515 is 175, key is test_row_0/A:col10/1733692497218/Put/seqid=0 2024-12-08T21:14:59,889 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49164 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:14:59,889 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/C is initiating minor compaction (all files) 2024-12-08T21:14:59,889 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/C in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:14:59,889 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/1e52b8ae16b6432fa4795ab3309d5e18, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d247707c91f40a2a9e9f3246a8e4bf5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/9f947fbb854148d9aa4b768bbc461505, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6a359cc1eb30433e97ab437c2c2da1fb] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=48.0 K 2024-12-08T21:14:59,890 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e52b8ae16b6432fa4795ab3309d5e18, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733692493693 2024-12-08T21:14:59,890 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d247707c91f40a2a9e9f3246a8e4bf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733692493868 2024-12-08T21:14:59,890 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f947fbb854148d9aa4b768bbc461505, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692495037 2024-12-08T21:14:59,890 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a359cc1eb30433e97ab437c2c2da1fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733692497210 2024-12-08T21:14:59,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742028_1204 (size=12454) 2024-12-08T21:14:59,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742029_1205 (size=31801) 2024-12-08T21:14:59,907 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:14:59,908 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/2493ca2fe3774699b5f877f3b52ebad4 is 50, key is test_row_0/C:col10/1733692497218/Put/seqid=0 2024-12-08T21:14:59,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742030_1206 (size=12847) 2024-12-08T21:14:59,922 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/2493ca2fe3774699b5f877f3b52ebad4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2493ca2fe3774699b5f877f3b52ebad4 2024-12-08T21:14:59,927 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/C of cddc5d594981d66d83e5fc7c39ab45c4 into 2493ca2fe3774699b5f877f3b52ebad4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:14:59,927 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:14:59,927 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/C, priority=12, startTime=1733692499844; duration=0sec 2024-12-08T21:14:59,927 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:14:59,927 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:C 2024-12-08T21:15:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T21:15:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:00,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. as already flushing 2024-12-08T21:15:00,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692560059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692560062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692560167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692560168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:00,298 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412080ccf3e8ff9ce41aead04fd4f37a56881_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080ccf3e8ff9ce41aead04fd4f37a56881_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:00,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2a4a3ac2122b48e9b5145358bf2783b7, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:15:00,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2a4a3ac2122b48e9b5145358bf2783b7 is 175, key is test_row_0/A:col10/1733692499414/Put/seqid=0 2024-12-08T21:15:00,301 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/655f6817e7ca43fe9f39934c558c6515 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/655f6817e7ca43fe9f39934c558c6515 2024-12-08T21:15:00,306 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/A of cddc5d594981d66d83e5fc7c39ab45c4 into 655f6817e7ca43fe9f39934c558c6515(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:00,306 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:15:00,306 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/A, priority=12, startTime=1733692499844; duration=0sec 2024-12-08T21:15:00,306 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:00,306 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:A 2024-12-08T21:15:00,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742031_1207 (size=31255) 2024-12-08T21:15:00,311 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2a4a3ac2122b48e9b5145358bf2783b7 2024-12-08T21:15:00,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/44f2a92e80994b119d56c6e17761aec4 is 50, key is test_row_0/B:col10/1733692499414/Put/seqid=0 2024-12-08T21:15:00,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742032_1208 (size=12301) 2024-12-08T21:15:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T21:15:00,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692560377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692560377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56612 deadline: 1733692560688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56608 deadline: 1733692560688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:00,722 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/44f2a92e80994b119d56c6e17761aec4 2024-12-08T21:15:00,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/10d5748e288b492186a68ea4129d392e is 50, key is test_row_0/C:col10/1733692499414/Put/seqid=0 2024-12-08T21:15:00,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742033_1209 (size=12301) 2024-12-08T21:15:00,780 INFO [Thread-691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-691. 2024-12-08T21:15:00,780 DEBUG [Thread-691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:00,780 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,780 DEBUG [Thread-691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:00,781 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-12-08T21:15:00,781 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:00,781 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,781 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:00,782 INFO [Thread-689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-689. 2024-12-08T21:15:00,782 DEBUG [Thread-689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:00,782 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,782 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:00,784 INFO [Thread-693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-693. 2024-12-08T21:15:00,784 DEBUG [Thread-693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:00,784 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,784 DEBUG [Thread-693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:00,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T21:15:01,133 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/10d5748e288b492186a68ea4129d392e 2024-12-08T21:15:01,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/2a4a3ac2122b48e9b5145358bf2783b7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2a4a3ac2122b48e9b5145358bf2783b7 2024-12-08T21:15:01,146 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2a4a3ac2122b48e9b5145358bf2783b7, entries=150, sequenceid=285, filesize=30.5 K 2024-12-08T21:15:01,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/44f2a92e80994b119d56c6e17761aec4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/44f2a92e80994b119d56c6e17761aec4 2024-12-08T21:15:01,152 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/44f2a92e80994b119d56c6e17761aec4, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T21:15:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/10d5748e288b492186a68ea4129d392e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/10d5748e288b492186a68ea4129d392e 2024-12-08T21:15:01,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/10d5748e288b492186a68ea4129d392e, entries=150, sequenceid=285, filesize=12.0 K 2024-12-08T21:15:01,159 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for cddc5d594981d66d83e5fc7c39ab45c4 in 1286ms, sequenceid=285, compaction requested=false 2024-12-08T21:15:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:15:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:01,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-08T21:15:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-08T21:15:01,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-08T21:15:01,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-12-08T21:15:01,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4440 sec 2024-12-08T21:15:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:01,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T21:15:01,199 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-12-08T21:15:01,199 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-12-08T21:15:01,199 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:01,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:15:01,199 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:01,199 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:01,199 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:01,199 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:01,199 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:01,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:01,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:01,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:15:01,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:01,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:15:01,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:01,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:01,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120841cc8f6fb50f4de98dbc0f50fe466711_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692501198/Put/seqid=0 2024-12-08T21:15:01,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742034_1210 (size=12454) 2024-12-08T21:15:01,609 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:01,614 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120841cc8f6fb50f4de98dbc0f50fe466711_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841cc8f6fb50f4de98dbc0f50fe466711_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:01,615 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3de8e5eecdf6494e874a1133e03e1db6, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:15:01,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3de8e5eecdf6494e874a1133e03e1db6 is 175, key is test_row_0/A:col10/1733692501198/Put/seqid=0 2024-12-08T21:15:01,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742035_1211 (size=31255) 2024-12-08T21:15:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-08T21:15:01,869 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:02,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3de8e5eecdf6494e874a1133e03e1db6 2024-12-08T21:15:02,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/7ecf24aca895452d9265e42d39ad736a is 50, key is test_row_0/B:col10/1733692501198/Put/seqid=0 2024-12-08T21:15:02,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742036_1212 (size=12301) 2024-12-08T21:15:02,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/7ecf24aca895452d9265e42d39ad736a 2024-12-08T21:15:02,442 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:15:02,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/8c416abca744459c9bf01a0f747850ab is 50, key is test_row_0/C:col10/1733692501198/Put/seqid=0 2024-12-08T21:15:02,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742037_1213 (size=12301) 2024-12-08T21:15:02,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/8c416abca744459c9bf01a0f747850ab 2024-12-08T21:15:02,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/3de8e5eecdf6494e874a1133e03e1db6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3de8e5eecdf6494e874a1133e03e1db6 2024-12-08T21:15:02,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3de8e5eecdf6494e874a1133e03e1db6, entries=150, sequenceid=303, filesize=30.5 K 2024-12-08T21:15:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/7ecf24aca895452d9265e42d39ad736a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/7ecf24aca895452d9265e42d39ad736a 2024-12-08T21:15:02,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/7ecf24aca895452d9265e42d39ad736a, entries=150, sequenceid=303, filesize=12.0 K 2024-12-08T21:15:02,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/8c416abca744459c9bf01a0f747850ab as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/8c416abca744459c9bf01a0f747850ab 2024-12-08T21:15:02,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/8c416abca744459c9bf01a0f747850ab, entries=150, sequenceid=303, filesize=12.0 K 2024-12-08T21:15:02,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for cddc5d594981d66d83e5fc7c39ab45c4 in 1689ms, sequenceid=303, compaction requested=true 2024-12-08T21:15:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:15:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:02,888 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cddc5d594981d66d83e5fc7c39ab45c4:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:02,888 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:02,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:02,889 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:02,889 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:02,889 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/A is initiating minor compaction (all files) 2024-12-08T21:15:02,889 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/B is initiating minor compaction (all files) 2024-12-08T21:15:02,889 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/B in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:02,889 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/A in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:02,890 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/f2c1cca3a10a4bcfbbf42af2b6de65cf, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/44f2a92e80994b119d56c6e17761aec4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/7ecf24aca895452d9265e42d39ad736a] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=36.6 K 2024-12-08T21:15:02,890 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/655f6817e7ca43fe9f39934c558c6515, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2a4a3ac2122b48e9b5145358bf2783b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3de8e5eecdf6494e874a1133e03e1db6] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=92.1 K 2024-12-08T21:15:02,890 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:02,890 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/655f6817e7ca43fe9f39934c558c6515, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2a4a3ac2122b48e9b5145358bf2783b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3de8e5eecdf6494e874a1133e03e1db6] 2024-12-08T21:15:02,890 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c1cca3a10a4bcfbbf42af2b6de65cf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733692497210 2024-12-08T21:15:02,890 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655f6817e7ca43fe9f39934c558c6515, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733692497210 2024-12-08T21:15:02,890 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f2a92e80994b119d56c6e17761aec4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733692499401 2024-12-08T21:15:02,890 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4a3ac2122b48e9b5145358bf2783b7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733692499401 2024-12-08T21:15:02,890 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ecf24aca895452d9265e42d39ad736a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733692500061 2024-12-08T21:15:02,891 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de8e5eecdf6494e874a1133e03e1db6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733692500061 2024-12-08T21:15:02,899 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#B#compaction#179 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:02,899 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/1cd939fa76ee482d8d1490d1c479f7d6 is 50, key is test_row_0/B:col10/1733692501198/Put/seqid=0 2024-12-08T21:15:02,901 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:15:02,905 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412088a8d30d4f9f343ab8457d2c7873b4cb2_cddc5d594981d66d83e5fc7c39ab45c4 store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:15:02,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742038_1214 (size=12949) 2024-12-08T21:15:02,921 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412088a8d30d4f9f343ab8457d2c7873b4cb2_cddc5d594981d66d83e5fc7c39ab45c4, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:15:02,922 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412088a8d30d4f9f343ab8457d2c7873b4cb2_cddc5d594981d66d83e5fc7c39ab45c4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:15:02,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742039_1215 (size=4469) 2024-12-08T21:15:03,313 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/1cd939fa76ee482d8d1490d1c479f7d6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/1cd939fa76ee482d8d1490d1c479f7d6 2024-12-08T21:15:03,320 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/B of cddc5d594981d66d83e5fc7c39ab45c4 into 1cd939fa76ee482d8d1490d1c479f7d6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:03,320 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:15:03,320 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/B, priority=13, startTime=1733692502888; duration=0sec 2024-12-08T21:15:03,320 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:03,320 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:B 2024-12-08T21:15:03,320 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:03,321 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:03,321 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): cddc5d594981d66d83e5fc7c39ab45c4/C is initiating minor compaction (all files) 2024-12-08T21:15:03,321 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cddc5d594981d66d83e5fc7c39ab45c4/C in TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:03,322 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2493ca2fe3774699b5f877f3b52ebad4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/10d5748e288b492186a68ea4129d392e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/8c416abca744459c9bf01a0f747850ab] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp, totalSize=36.6 K 2024-12-08T21:15:03,322 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 2493ca2fe3774699b5f877f3b52ebad4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733692497210 2024-12-08T21:15:03,322 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 10d5748e288b492186a68ea4129d392e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733692499401 2024-12-08T21:15:03,323 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c416abca744459c9bf01a0f747850ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733692500061 2024-12-08T21:15:03,327 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#A#compaction#180 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:03,327 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/506f80ba8bb64843941e1107fe897f24 is 175, key is test_row_0/A:col10/1733692501198/Put/seqid=0 2024-12-08T21:15:03,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742040_1216 (size=31903) 2024-12-08T21:15:03,334 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cddc5d594981d66d83e5fc7c39ab45c4#C#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:03,334 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6424f89a143c4637838747dd8509ea93 is 50, key is test_row_0/C:col10/1733692501198/Put/seqid=0 2024-12-08T21:15:03,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742041_1217 (size=12949) 2024-12-08T21:15:03,743 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/506f80ba8bb64843941e1107fe897f24 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/506f80ba8bb64843941e1107fe897f24 2024-12-08T21:15:03,744 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/6424f89a143c4637838747dd8509ea93 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6424f89a143c4637838747dd8509ea93 2024-12-08T21:15:03,747 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/A of cddc5d594981d66d83e5fc7c39ab45c4 into 506f80ba8bb64843941e1107fe897f24(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:03,747 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:15:03,747 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/A, priority=13, startTime=1733692502888; duration=0sec 2024-12-08T21:15:03,748 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:03,748 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:A 2024-12-08T21:15:03,749 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cddc5d594981d66d83e5fc7c39ab45c4/C of cddc5d594981d66d83e5fc7c39ab45c4 into 6424f89a143c4637838747dd8509ea93(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:03,749 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cddc5d594981d66d83e5fc7c39ab45c4: 2024-12-08T21:15:03,749 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., storeName=cddc5d594981d66d83e5fc7c39ab45c4/C, priority=13, startTime=1733692502888; duration=0sec 2024-12-08T21:15:03,749 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:03,749 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cddc5d594981d66d83e5fc7c39ab45c4:C 2024-12-08T21:15:08,279 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-12-08T21:15:08,279 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:08,279 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,280 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:08,290 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-12-08T21:15:08,290 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-12-08T21:15:08,290 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:08,290 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:08,290 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,290 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,290 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,290 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:08,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4240 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4340 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1863 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5587 rows 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1858 2024-12-08T21:15:08,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5571 rows 2024-12-08T21:15:08,290 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:15:08,290 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:15:08,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,290 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:08,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:08,291 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T21:15:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T21:15:08,294 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692508294"}]},"ts":"1733692508294"} 2024-12-08T21:15:08,295 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T21:15:08,295 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T21:15:08,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:15:08,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, UNASSIGN}] 2024-12-08T21:15:08,297 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, UNASSIGN 2024-12-08T21:15:08,298 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:08,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, UNASSIGN because future has completed 2024-12-08T21:15:08,300 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:15:08,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:15:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T21:15:08,452 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:08,452 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing cddc5d594981d66d83e5fc7c39ab45c4, disabling compactions & flushes 2024-12-08T21:15:08,453 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. after waiting 0 ms 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:08,453 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing cddc5d594981d66d83e5fc7c39ab45c4 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=A 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=B 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cddc5d594981d66d83e5fc7c39ab45c4, store=C 2024-12-08T21:15:08,453 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:08,460 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085e1bc9b6fd6b4b049d94e9fd15afb694_cddc5d594981d66d83e5fc7c39ab45c4 is 50, key is test_row_0/A:col10/1733692508288/Put/seqid=0 2024-12-08T21:15:08,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742042_1218 (size=9914) 2024-12-08T21:15:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T21:15:08,869 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:08,878 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085e1bc9b6fd6b4b049d94e9fd15afb694_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085e1bc9b6fd6b4b049d94e9fd15afb694_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:08,879 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e2a620bc1cab4f1dba4ccf6729e4fdb6, store: [table=TestAcidGuarantees family=A region=cddc5d594981d66d83e5fc7c39ab45c4] 2024-12-08T21:15:08,880 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e2a620bc1cab4f1dba4ccf6729e4fdb6 is 175, key is test_row_0/A:col10/1733692508288/Put/seqid=0 2024-12-08T21:15:08,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742043_1219 (size=22561) 2024-12-08T21:15:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T21:15:09,287 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e2a620bc1cab4f1dba4ccf6729e4fdb6 2024-12-08T21:15:09,293 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/ef4eda3e04224b19bc5b3b0db62ffc00 is 50, key is test_row_0/B:col10/1733692508288/Put/seqid=0 2024-12-08T21:15:09,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742044_1220 (size=9857) 2024-12-08T21:15:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T21:15:09,698 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/ef4eda3e04224b19bc5b3b0db62ffc00 2024-12-08T21:15:09,705 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/b654efe011084d70ac06e1efb0742a31 is 50, key is test_row_0/C:col10/1733692508288/Put/seqid=0 2024-12-08T21:15:09,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742045_1221 (size=9857) 2024-12-08T21:15:10,110 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/b654efe011084d70ac06e1efb0742a31 2024-12-08T21:15:10,116 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/A/e2a620bc1cab4f1dba4ccf6729e4fdb6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e2a620bc1cab4f1dba4ccf6729e4fdb6 2024-12-08T21:15:10,121 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e2a620bc1cab4f1dba4ccf6729e4fdb6, entries=100, sequenceid=312, filesize=22.0 K 2024-12-08T21:15:10,122 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/B/ef4eda3e04224b19bc5b3b0db62ffc00 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ef4eda3e04224b19bc5b3b0db62ffc00 2024-12-08T21:15:10,126 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ef4eda3e04224b19bc5b3b0db62ffc00, entries=100, sequenceid=312, filesize=9.6 K 2024-12-08T21:15:10,127 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/.tmp/C/b654efe011084d70ac06e1efb0742a31 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/b654efe011084d70ac06e1efb0742a31 2024-12-08T21:15:10,130 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/b654efe011084d70ac06e1efb0742a31, entries=100, sequenceid=312, filesize=9.6 K 2024-12-08T21:15:10,131 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for cddc5d594981d66d83e5fc7c39ab45c4 in 1678ms, sequenceid=312, compaction requested=false 2024-12-08T21:15:10,131 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e0f9bdc9acc548c79a044d12d3644a1e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/4e08eb2502ca44e6a214ae120e94e145, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a2eec19589414799bac341e2322f1857, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/11b7610b910c4e14a0d21bcd22a39093, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1429c61f9a51454db3db1540196f719a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/07a6dfdf397f4a7e842f593c554c7c36, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3b1ad5ebee634a658e4628c81902f62e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/f34d9e02a04d432f94f8591386256e15, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c4ede7df97214255b4c9aad2287bb9ae, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/6841c2996c034f82a9c3424cd4d79840, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2d13cd9d436f4fd2a8988edac6632dde, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a8e2fad7ea904befbbef5816aabe2f3a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c7201d6eb1d44b95930b74dc0c04e815, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/380a42aeae724b739af875a2a51ddb21, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1e25bda5ce34414798d871f00d2d2dfe, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3c20d34fcc544d6e99a1f62a956c0047, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/655f6817e7ca43fe9f39934c558c6515, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2a4a3ac2122b48e9b5145358bf2783b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3de8e5eecdf6494e874a1133e03e1db6] to archive 2024-12-08T21:15:10,132 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:15:10,133 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e0f9bdc9acc548c79a044d12d3644a1e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e0f9bdc9acc548c79a044d12d3644a1e 2024-12-08T21:15:10,134 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/4e08eb2502ca44e6a214ae120e94e145 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/4e08eb2502ca44e6a214ae120e94e145 2024-12-08T21:15:10,135 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a2eec19589414799bac341e2322f1857 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a2eec19589414799bac341e2322f1857 2024-12-08T21:15:10,136 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/11b7610b910c4e14a0d21bcd22a39093 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/11b7610b910c4e14a0d21bcd22a39093 2024-12-08T21:15:10,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1429c61f9a51454db3db1540196f719a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1429c61f9a51454db3db1540196f719a 2024-12-08T21:15:10,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/07a6dfdf397f4a7e842f593c554c7c36 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/07a6dfdf397f4a7e842f593c554c7c36 2024-12-08T21:15:10,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3b1ad5ebee634a658e4628c81902f62e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3b1ad5ebee634a658e4628c81902f62e 2024-12-08T21:15:10,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/f34d9e02a04d432f94f8591386256e15 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/f34d9e02a04d432f94f8591386256e15 2024-12-08T21:15:10,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c4ede7df97214255b4c9aad2287bb9ae to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c4ede7df97214255b4c9aad2287bb9ae 2024-12-08T21:15:10,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/6841c2996c034f82a9c3424cd4d79840 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/6841c2996c034f82a9c3424cd4d79840 2024-12-08T21:15:10,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2d13cd9d436f4fd2a8988edac6632dde to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2d13cd9d436f4fd2a8988edac6632dde 2024-12-08T21:15:10,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a8e2fad7ea904befbbef5816aabe2f3a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/a8e2fad7ea904befbbef5816aabe2f3a 2024-12-08T21:15:10,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c7201d6eb1d44b95930b74dc0c04e815 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/c7201d6eb1d44b95930b74dc0c04e815 2024-12-08T21:15:10,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/380a42aeae724b739af875a2a51ddb21 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/380a42aeae724b739af875a2a51ddb21 2024-12-08T21:15:10,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1e25bda5ce34414798d871f00d2d2dfe to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/1e25bda5ce34414798d871f00d2d2dfe 2024-12-08T21:15:10,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3c20d34fcc544d6e99a1f62a956c0047 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3c20d34fcc544d6e99a1f62a956c0047 2024-12-08T21:15:10,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/655f6817e7ca43fe9f39934c558c6515 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/655f6817e7ca43fe9f39934c558c6515 2024-12-08T21:15:10,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2a4a3ac2122b48e9b5145358bf2783b7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/2a4a3ac2122b48e9b5145358bf2783b7 2024-12-08T21:15:10,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3de8e5eecdf6494e874a1133e03e1db6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/3de8e5eecdf6494e874a1133e03e1db6 2024-12-08T21:15:10,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/e362f557e7be40b1952aba5729837260, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8f0aaa426d8d4006886c7faf1719982c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c4e001cebd884f3f9f9b9c19fb698303, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a33c4d13800d41f1ac3e586886a566b3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a2e9708aaef54f458d719e757c0b5892, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/aef208f8a144487dbb22fbaddf2b0a44, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8714ed7378a14d2c87d968f1b2cdfe8c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/bf4392816ef44e6d997b384200a38f51, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c29c192b18cd4550976ea195d5abf3be, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ce3835fb763b47a4acbc4211a7106578, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/108571a3393043ad9c5a6891093eefc8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/459b149c2c2440d587ffecc09db6d5c4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/549ce694c3f94902a556a3db4607478e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/cc1020f69a764ca88e78700c7700b32b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/3a683409302b488986518604a00dd5ee, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/f2c1cca3a10a4bcfbbf42af2b6de65cf, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/dc23cd7fb3394ce782e3c737c998b7c5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/44f2a92e80994b119d56c6e17761aec4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/7ecf24aca895452d9265e42d39ad736a] to archive 2024-12-08T21:15:10,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:15:10,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/e362f557e7be40b1952aba5729837260 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/e362f557e7be40b1952aba5729837260 2024-12-08T21:15:10,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8f0aaa426d8d4006886c7faf1719982c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8f0aaa426d8d4006886c7faf1719982c 2024-12-08T21:15:10,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c4e001cebd884f3f9f9b9c19fb698303 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c4e001cebd884f3f9f9b9c19fb698303 2024-12-08T21:15:10,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a33c4d13800d41f1ac3e586886a566b3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a33c4d13800d41f1ac3e586886a566b3 2024-12-08T21:15:10,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a2e9708aaef54f458d719e757c0b5892 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/a2e9708aaef54f458d719e757c0b5892 2024-12-08T21:15:10,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/aef208f8a144487dbb22fbaddf2b0a44 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/aef208f8a144487dbb22fbaddf2b0a44 2024-12-08T21:15:10,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8714ed7378a14d2c87d968f1b2cdfe8c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/8714ed7378a14d2c87d968f1b2cdfe8c 2024-12-08T21:15:10,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/bf4392816ef44e6d997b384200a38f51 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/bf4392816ef44e6d997b384200a38f51 2024-12-08T21:15:10,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c29c192b18cd4550976ea195d5abf3be to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/c29c192b18cd4550976ea195d5abf3be 2024-12-08T21:15:10,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ce3835fb763b47a4acbc4211a7106578 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ce3835fb763b47a4acbc4211a7106578 2024-12-08T21:15:10,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/108571a3393043ad9c5a6891093eefc8 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/108571a3393043ad9c5a6891093eefc8 2024-12-08T21:15:10,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/459b149c2c2440d587ffecc09db6d5c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/459b149c2c2440d587ffecc09db6d5c4 2024-12-08T21:15:10,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/549ce694c3f94902a556a3db4607478e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/549ce694c3f94902a556a3db4607478e 2024-12-08T21:15:10,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/cc1020f69a764ca88e78700c7700b32b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/cc1020f69a764ca88e78700c7700b32b 2024-12-08T21:15:10,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/3a683409302b488986518604a00dd5ee to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/3a683409302b488986518604a00dd5ee 2024-12-08T21:15:10,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/f2c1cca3a10a4bcfbbf42af2b6de65cf to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/f2c1cca3a10a4bcfbbf42af2b6de65cf 2024-12-08T21:15:10,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/dc23cd7fb3394ce782e3c737c998b7c5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/dc23cd7fb3394ce782e3c737c998b7c5 2024-12-08T21:15:10,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/44f2a92e80994b119d56c6e17761aec4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/44f2a92e80994b119d56c6e17761aec4 2024-12-08T21:15:10,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/7ecf24aca895452d9265e42d39ad736a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/7ecf24aca895452d9265e42d39ad736a 2024-12-08T21:15:10,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/dfa7f77d079349d69dcf1be962d79693, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5622d57b10704d4d8e0f822c2e67a256, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5d0b95429cf74e67bba23f984f00e06e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/cca401ec00e5402eba85033bfe189f01, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5c444a729ee0434f9dbd086ca686017d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/76c3686a87df4c8693a5f7ff7dea72bc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6dc317ec7b3f4d81b5f4eb804d52fd95, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d9fccbcec2f4632b3ff70b68fbd6241, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2ec1d571e84946ddb8612cf559d29ddf, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6759bad476324ed2bc62109b9be9600d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6ed5b757f2d643a788d537ee237ae296, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/1e52b8ae16b6432fa4795ab3309d5e18, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/82b195f759f145b2be149c7cd404ba6f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d247707c91f40a2a9e9f3246a8e4bf5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/9f947fbb854148d9aa4b768bbc461505, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2493ca2fe3774699b5f877f3b52ebad4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6a359cc1eb30433e97ab437c2c2da1fb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/10d5748e288b492186a68ea4129d392e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/8c416abca744459c9bf01a0f747850ab] to archive 2024-12-08T21:15:10,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:15:10,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/dfa7f77d079349d69dcf1be962d79693 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/dfa7f77d079349d69dcf1be962d79693 2024-12-08T21:15:10,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5622d57b10704d4d8e0f822c2e67a256 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5622d57b10704d4d8e0f822c2e67a256 2024-12-08T21:15:10,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5d0b95429cf74e67bba23f984f00e06e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5d0b95429cf74e67bba23f984f00e06e 2024-12-08T21:15:10,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/cca401ec00e5402eba85033bfe189f01 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/cca401ec00e5402eba85033bfe189f01 2024-12-08T21:15:10,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5c444a729ee0434f9dbd086ca686017d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/5c444a729ee0434f9dbd086ca686017d 2024-12-08T21:15:10,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/76c3686a87df4c8693a5f7ff7dea72bc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/76c3686a87df4c8693a5f7ff7dea72bc 2024-12-08T21:15:10,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6dc317ec7b3f4d81b5f4eb804d52fd95 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6dc317ec7b3f4d81b5f4eb804d52fd95 2024-12-08T21:15:10,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d9fccbcec2f4632b3ff70b68fbd6241 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d9fccbcec2f4632b3ff70b68fbd6241 2024-12-08T21:15:10,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2ec1d571e84946ddb8612cf559d29ddf to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2ec1d571e84946ddb8612cf559d29ddf 2024-12-08T21:15:10,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6759bad476324ed2bc62109b9be9600d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6759bad476324ed2bc62109b9be9600d 2024-12-08T21:15:10,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6ed5b757f2d643a788d537ee237ae296 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6ed5b757f2d643a788d537ee237ae296 2024-12-08T21:15:10,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/1e52b8ae16b6432fa4795ab3309d5e18 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/1e52b8ae16b6432fa4795ab3309d5e18 2024-12-08T21:15:10,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/82b195f759f145b2be149c7cd404ba6f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/82b195f759f145b2be149c7cd404ba6f 2024-12-08T21:15:10,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d247707c91f40a2a9e9f3246a8e4bf5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/0d247707c91f40a2a9e9f3246a8e4bf5 2024-12-08T21:15:10,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/9f947fbb854148d9aa4b768bbc461505 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/9f947fbb854148d9aa4b768bbc461505 2024-12-08T21:15:10,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2493ca2fe3774699b5f877f3b52ebad4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/2493ca2fe3774699b5f877f3b52ebad4 2024-12-08T21:15:10,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6a359cc1eb30433e97ab437c2c2da1fb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6a359cc1eb30433e97ab437c2c2da1fb 2024-12-08T21:15:10,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/10d5748e288b492186a68ea4129d392e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/10d5748e288b492186a68ea4129d392e 2024-12-08T21:15:10,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/8c416abca744459c9bf01a0f747850ab to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/8c416abca744459c9bf01a0f747850ab 2024-12-08T21:15:10,187 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/recovered.edits/315.seqid, newMaxSeqId=315, maxSeqId=4 2024-12-08T21:15:10,188 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. 2024-12-08T21:15:10,188 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for cddc5d594981d66d83e5fc7c39ab45c4: Waiting for close lock at 1733692508452Running coprocessor pre-close hooks at 1733692508452Disabling compacts and flushes for region at 1733692508452Disabling writes for close at 1733692508453 (+1 ms)Obtaining lock to block concurrent updates at 1733692508453Preparing flush snapshotting stores in cddc5d594981d66d83e5fc7c39ab45c4 at 1733692508453Finished memstore snapshotting TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1733692508453Flushing stores of TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4. at 1733692508454 (+1 ms)Flushing cddc5d594981d66d83e5fc7c39ab45c4/A: creating writer at 1733692508454Flushing mob file cddc5d594981d66d83e5fc7c39ab45c4/A: appending metadata at 1733692508460 (+6 ms)Flushing mob file cddc5d594981d66d83e5fc7c39ab45c4/A: closing flushed file at 1733692508460Flushing cddc5d594981d66d83e5fc7c39ab45c4/A: appending metadata at 1733692508880 (+420 ms)Flushing cddc5d594981d66d83e5fc7c39ab45c4/A: closing flushed file at 1733692508880Flushing cddc5d594981d66d83e5fc7c39ab45c4/B: creating writer at 1733692509290 (+410 ms)Flushing cddc5d594981d66d83e5fc7c39ab45c4/B: appending metadata at 1733692509292 (+2 ms)Flushing cddc5d594981d66d83e5fc7c39ab45c4/B: closing flushed file at 1733692509292Flushing cddc5d594981d66d83e5fc7c39ab45c4/C: creating writer at 1733692509702 (+410 ms)Flushing cddc5d594981d66d83e5fc7c39ab45c4/C: appending metadata at 1733692509705 (+3 ms)Flushing cddc5d594981d66d83e5fc7c39ab45c4/C: closing flushed file at 1733692509705Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fa3c558: reopening flushed file at 1733692510115 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72e08ea1: reopening flushed file at 1733692510121 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45092e44: reopening flushed file at 1733692510126 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for cddc5d594981d66d83e5fc7c39ab45c4 in 1678ms, sequenceid=312, compaction requested=false at 1733692510131 (+5 ms)Writing region close event to WAL at 1733692510185 (+54 ms)Running coprocessor post-close hooks at 1733692510188 (+3 ms)Closed at 1733692510188 2024-12-08T21:15:10,189 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,190 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=cddc5d594981d66d83e5fc7c39ab45c4, regionState=CLOSED 2024-12-08T21:15:10,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:15:10,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-08T21:15:10,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure cddc5d594981d66d83e5fc7c39ab45c4, server=e653e208f502,38457,1733692444075 in 1.8910 sec 2024-12-08T21:15:10,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-08T21:15:10,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cddc5d594981d66d83e5fc7c39ab45c4, UNASSIGN in 1.8960 sec 2024-12-08T21:15:10,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-08T21:15:10,195 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8990 sec 2024-12-08T21:15:10,196 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692510196"}]},"ts":"1733692510196"} 2024-12-08T21:15:10,197 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T21:15:10,198 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T21:15:10,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9070 sec 2024-12-08T21:15:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-08T21:15:10,438 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:10,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T21:15:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:10,440 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T21:15:10,441 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:10,444 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,446 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/recovered.edits] 2024-12-08T21:15:10,449 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/506f80ba8bb64843941e1107fe897f24 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/506f80ba8bb64843941e1107fe897f24 2024-12-08T21:15:10,450 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e2a620bc1cab4f1dba4ccf6729e4fdb6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/A/e2a620bc1cab4f1dba4ccf6729e4fdb6 2024-12-08T21:15:10,452 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/1cd939fa76ee482d8d1490d1c479f7d6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/1cd939fa76ee482d8d1490d1c479f7d6 2024-12-08T21:15:10,454 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ef4eda3e04224b19bc5b3b0db62ffc00 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/B/ef4eda3e04224b19bc5b3b0db62ffc00 2024-12-08T21:15:10,456 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6424f89a143c4637838747dd8509ea93 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/6424f89a143c4637838747dd8509ea93 2024-12-08T21:15:10,457 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/b654efe011084d70ac06e1efb0742a31 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/C/b654efe011084d70ac06e1efb0742a31 2024-12-08T21:15:10,460 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/recovered.edits/315.seqid to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4/recovered.edits/315.seqid 2024-12-08T21:15:10,461 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,461 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T21:15:10,461 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T21:15:10,462 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T21:15:10,467 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080ccf3e8ff9ce41aead04fd4f37a56881_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412080ccf3e8ff9ce41aead04fd4f37a56881_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,468 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084155c28dba274776a6d12407e1af888e_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084155c28dba274776a6d12407e1af888e_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,470 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841cc8f6fb50f4de98dbc0f50fe466711_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120841cc8f6fb50f4de98dbc0f50fe466711_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,471 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120851540a04cd3e4fbb9088e3590fc3f9e2_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120851540a04cd3e4fbb9088e3590fc3f9e2_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,473 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085ad7705e28c34e6a9b594a528a19bafb_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085ad7705e28c34e6a9b594a528a19bafb_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,474 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085e1bc9b6fd6b4b049d94e9fd15afb694_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085e1bc9b6fd6b4b049d94e9fd15afb694_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,476 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208675d18ed779844768bc555b866e8c86f_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208675d18ed779844768bc555b866e8c86f_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,477 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088456d1664b24496dabb72783297b6eea_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412088456d1664b24496dabb72783297b6eea_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,478 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208879c524b42744645b82b6f5be4bb6a44_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208879c524b42744645b82b6f5be4bb6a44_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,479 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208893ce80d98b04323a66a907ee371a7c1_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208893ce80d98b04323a66a907ee371a7c1_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,481 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120899595b5a668641c6afa8d6cf9a2d8f2c_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120899595b5a668641c6afa8d6cf9a2d8f2c_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,482 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208be5803f3a5af407fbe2105d1bb9def05_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208be5803f3a5af407fbe2105d1bb9def05_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,483 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c6171b9f4b1f4812b0ead6a09f108b93_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c6171b9f4b1f4812b0ead6a09f108b93_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,485 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c657ecb3e238467995ed74f49f16ab6a_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c657ecb3e238467995ed74f49f16ab6a_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,486 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d5ab0f9d110246df98c2c4e0c86ed572_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d5ab0f9d110246df98c2c4e0c86ed572_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,487 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d709cf52792d45d081b147d8495e29b3_cddc5d594981d66d83e5fc7c39ab45c4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208d709cf52792d45d081b147d8495e29b3_cddc5d594981d66d83e5fc7c39ab45c4 2024-12-08T21:15:10,487 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T21:15:10,489 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:10,492 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T21:15:10,493 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T21:15:10,495 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:10,495 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T21:15:10,495 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733692510495"}]},"ts":"9223372036854775807"} 2024-12-08T21:15:10,497 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T21:15:10,497 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cddc5d594981d66d83e5fc7c39ab45c4, NAME => 'TestAcidGuarantees,,1733692477599.cddc5d594981d66d83e5fc7c39ab45c4.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T21:15:10,497 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T21:15:10,497 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733692510497"}]},"ts":"9223372036854775807"} 2024-12-08T21:15:10,499 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T21:15:10,499 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:10,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-12-08T21:15:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-08T21:15:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T21:15:10,548 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:10,555 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 250), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=373 (was 292) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8637 (was 7765) - AvailableMemoryMB LEAK? - 2024-12-08T21:15:10,562 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=373, ProcessCount=11, AvailableMemoryMB=8636 2024-12-08T21:15:10,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:15:10,563 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:15:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:10,565 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:15:10,565 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:10,565 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-08T21:15:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T21:15:10,566 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:15:10,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742046_1222 (size=1061) 2024-12-08T21:15:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T21:15:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T21:15:10,974 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:15:10,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742047_1223 (size=53) 2024-12-08T21:15:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T21:15:11,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:15:11,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b7d9f8874d392b78c0ba8e541e909290, disabling compactions & flushes 2024-12-08T21:15:11,380 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. after waiting 0 ms 2024-12-08T21:15:11,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,380 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b7d9f8874d392b78c0ba8e541e909290: Waiting for close lock at 1733692511380Disabling compacts and flushes for region at 1733692511380Disabling writes for close at 1733692511380Writing region close event to WAL at 1733692511380Closed at 1733692511380 2024-12-08T21:15:11,382 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:15:11,382 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733692511382"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733692511382"}]},"ts":"1733692511382"} 2024-12-08T21:15:11,383 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:15:11,384 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:15:11,384 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692511384"}]},"ts":"1733692511384"} 2024-12-08T21:15:11,386 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T21:15:11,386 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, ASSIGN}] 2024-12-08T21:15:11,387 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, ASSIGN 2024-12-08T21:15:11,388 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, ASSIGN; state=OFFLINE, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=false 2024-12-08T21:15:11,539 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=b7d9f8874d392b78c0ba8e541e909290, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:11,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, ASSIGN because future has completed 2024-12-08T21:15:11,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:15:11,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T21:15:11,705 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,706 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:15:11,706 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,706 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:15:11,707 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,707 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,710 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,713 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:11,713 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d9f8874d392b78c0ba8e541e909290 columnFamilyName A 2024-12-08T21:15:11,713 DEBUG [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:11,715 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.HStore(327): Store=b7d9f8874d392b78c0ba8e541e909290/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:11,715 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,716 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:11,716 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d9f8874d392b78c0ba8e541e909290 columnFamilyName B 2024-12-08T21:15:11,716 DEBUG [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:11,717 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.HStore(327): Store=b7d9f8874d392b78c0ba8e541e909290/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:11,717 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,718 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:11,718 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7d9f8874d392b78c0ba8e541e909290 columnFamilyName C 2024-12-08T21:15:11,719 DEBUG [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:11,719 INFO [StoreOpener-b7d9f8874d392b78c0ba8e541e909290-1 {}] regionserver.HStore(327): Store=b7d9f8874d392b78c0ba8e541e909290/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:11,719 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,719 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,720 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,721 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,722 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,722 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,723 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:15:11,724 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,727 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:15:11,727 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened b7d9f8874d392b78c0ba8e541e909290; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58976138, jitterRate=-0.12118706107139587}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:15:11,727 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:11,728 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for b7d9f8874d392b78c0ba8e541e909290: Running coprocessor pre-open hook at 1733692511707Writing region info on filesystem at 1733692511707Initializing all the Stores at 1733692511709 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692511709Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692511709Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692511709Cleaning up temporary data from old regions at 1733692511722 (+13 ms)Running coprocessor post-open hooks at 1733692511727 (+5 ms)Region opened successfully at 1733692511728 (+1 ms) 2024-12-08T21:15:11,729 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., pid=61, masterSystemTime=1733692511698 2024-12-08T21:15:11,731 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,731 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:11,731 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=b7d9f8874d392b78c0ba8e541e909290, regionState=OPEN, openSeqNum=2, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:11,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:15:11,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-08T21:15:11,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 in 189 msec 2024-12-08T21:15:11,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-08T21:15:11,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, ASSIGN in 349 msec 2024-12-08T21:15:11,737 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:15:11,737 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692511737"}]},"ts":"1733692511737"} 2024-12-08T21:15:11,739 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T21:15:11,739 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:15:11,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-12-08T21:15:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-08T21:15:12,709 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:12,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555e4e20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a7d304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d72d05c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,724 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:15:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e653e208f502,45981,1733692443391 2024-12-08T21:15:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c187f19 2024-12-08T21:15:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:15:12,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:15:12,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ad36e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,728 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6920585c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d168a27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b38a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4655b440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,734 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e209bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260eab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,735 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f571d63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,737 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d39646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699a7f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,738 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@186beea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,739 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1cfce4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,742 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20bab93a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,743 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d390f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,745 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a76ac4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,746 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc1f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,748 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@124428c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,749 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13acb63e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,751 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688037bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,752 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcaeeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,754 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8885b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,755 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c6befb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:12,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:12,757 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f09c3d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:12,758 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bbdf15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:12,759 DEBUG [Thread-1063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,759 DEBUG [Thread-1065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,760 DEBUG [Thread-1052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,760 DEBUG [Thread-1069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,760 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,761 DEBUG [Thread-1054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-08T21:15:12,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T21:15:12,762 DEBUG [Thread-1056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,763 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,763 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,763 DEBUG [Thread-1058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,764 DEBUG [Thread-1067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,765 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:12,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:12,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:15:12,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:12,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:15:12,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:12,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:12,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:12,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:12,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:12,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:12,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692572806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/47d00e4576724f978b89a60e55ca23e9 is 50, key is test_row_0/A:col10/1733692512777/Put/seqid=0 2024-12-08T21:15:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692512794, max timestamp=1733692512813Num uniques 859; 2024-12-08T21:15:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692512794, max timestamp=1733692512813Num uniques 859; 2024-12-08T21:15:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:15:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:15:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692512794, max timestamp=1733692512813Num uniques 859; 2024-12-08T21:15:12,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:15:12,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692572836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692572842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692572844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692572844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742048_1224 (size=14341) 2024-12-08T21:15:12,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/47d00e4576724f978b89a60e55ca23e9 2024-12-08T21:15:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T21:15:12,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/ccbddf5acaa244f0b27f56343b64c741 is 50, key is test_row_0/B:col10/1733692512777/Put/seqid=0 2024-12-08T21:15:12,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742049_1225 (size=12001) 2024-12-08T21:15:12,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/ccbddf5acaa244f0b27f56343b64c741 2024-12-08T21:15:12,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T21:15:12,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:12,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:12,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:12,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:12,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:12,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692572919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692572948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692572959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692572959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692572959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:12,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/cd0f60d0f95e4cc5afccaa8dd1e27316 is 50, key is test_row_0/C:col10/1733692512777/Put/seqid=0 2024-12-08T21:15:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742050_1226 (size=12001) 2024-12-08T21:15:13,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/cd0f60d0f95e4cc5afccaa8dd1e27316 2024-12-08T21:15:13,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/47d00e4576724f978b89a60e55ca23e9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/47d00e4576724f978b89a60e55ca23e9 2024-12-08T21:15:13,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/47d00e4576724f978b89a60e55ca23e9, entries=200, sequenceid=12, filesize=14.0 K 2024-12-08T21:15:13,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/ccbddf5acaa244f0b27f56343b64c741 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ccbddf5acaa244f0b27f56343b64c741 2024-12-08T21:15:13,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ccbddf5acaa244f0b27f56343b64c741, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T21:15:13,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/cd0f60d0f95e4cc5afccaa8dd1e27316 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/cd0f60d0f95e4cc5afccaa8dd1e27316 2024-12-08T21:15:13,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/cd0f60d0f95e4cc5afccaa8dd1e27316, entries=150, sequenceid=12, filesize=11.7 K 2024-12-08T21:15:13,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b7d9f8874d392b78c0ba8e541e909290 in 279ms, sequenceid=12, compaction requested=false 2024-12-08T21:15:13,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:13,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-08T21:15:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:13,075 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-08T21:15:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:13,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:13,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T21:15:13,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/cd991e8624e54e068950add7827501f8 is 50, key is test_row_0/A:col10/1733692512834/Put/seqid=0 2024-12-08T21:15:13,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742051_1227 (size=12001) 2024-12-08T21:15:13,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/cd991e8624e54e068950add7827501f8 2024-12-08T21:15:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:13,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:13,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/b9169cd519a74089aade947246ca846f is 50, key is test_row_0/B:col10/1733692512834/Put/seqid=0 2024-12-08T21:15:13,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742052_1228 (size=12001) 2024-12-08T21:15:13,162 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/b9169cd519a74089aade947246ca846f 2024-12-08T21:15:13,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692573168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692573169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692573169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692573169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692573173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/151aee0698414259bc31d0af3ae8e078 is 50, key is test_row_0/C:col10/1733692512834/Put/seqid=0 2024-12-08T21:15:13,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742053_1229 (size=12001) 2024-12-08T21:15:13,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692573278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692573288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T21:15:13,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692573488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692573488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692573488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692573488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692573497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,601 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/151aee0698414259bc31d0af3ae8e078 2024-12-08T21:15:13,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/cd991e8624e54e068950add7827501f8 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/cd991e8624e54e068950add7827501f8 2024-12-08T21:15:13,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/cd991e8624e54e068950add7827501f8, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T21:15:13,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/b9169cd519a74089aade947246ca846f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/b9169cd519a74089aade947246ca846f 2024-12-08T21:15:13,620 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/b9169cd519a74089aade947246ca846f, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T21:15:13,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/151aee0698414259bc31d0af3ae8e078 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/151aee0698414259bc31d0af3ae8e078 2024-12-08T21:15:13,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/151aee0698414259bc31d0af3ae8e078, entries=150, sequenceid=46, filesize=11.7 K 2024-12-08T21:15:13,635 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for b7d9f8874d392b78c0ba8e541e909290 in 560ms, sequenceid=46, compaction requested=false 2024-12-08T21:15:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-08T21:15:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-08T21:15:13,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-08T21:15:13,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 872 msec 2024-12-08T21:15:13,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 883 msec 2024-12-08T21:15:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:13,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:15:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:13,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/475222b469c04dd48d3dfc86807d33e8 is 50, key is test_row_0/A:col10/1733692513169/Put/seqid=0 2024-12-08T21:15:13,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742054_1230 (size=12001) 2024-12-08T21:15:13,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692573831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692573832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-08T21:15:13,898 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:13,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-08T21:15:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T21:15:13,900 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:13,901 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:13,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692573938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692573938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692573998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692573998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692574001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T21:15:14,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:14,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:14,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692574148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692574148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/475222b469c04dd48d3dfc86807d33e8 2024-12-08T21:15:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T21:15:14,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/029eeef15ee943429ffae6fab4d1b4e9 is 50, key is test_row_0/B:col10/1733692513169/Put/seqid=0 2024-12-08T21:15:14,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742055_1231 (size=12001) 2024-12-08T21:15:14,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692574458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692574458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T21:15:14,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/029eeef15ee943429ffae6fab4d1b4e9 2024-12-08T21:15:14,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/354ece3f707544eaa5f749749c5763b2 is 50, key is test_row_0/C:col10/1733692513169/Put/seqid=0 2024-12-08T21:15:14,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742056_1232 (size=12001) 2024-12-08T21:15:14,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:14,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:14,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692574968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692574968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:14,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:14,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:14,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:14,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692575007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:15,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692575007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:15,009 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:15,010 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:15,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:15,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692575018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:15,019 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:15,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T21:15:15,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/354ece3f707544eaa5f749749c5763b2 2024-12-08T21:15:15,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/475222b469c04dd48d3dfc86807d33e8 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/475222b469c04dd48d3dfc86807d33e8 2024-12-08T21:15:15,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/475222b469c04dd48d3dfc86807d33e8, entries=150, sequenceid=58, filesize=11.7 K 2024-12-08T21:15:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/029eeef15ee943429ffae6fab4d1b4e9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/029eeef15ee943429ffae6fab4d1b4e9 2024-12-08T21:15:15,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/029eeef15ee943429ffae6fab4d1b4e9, entries=150, sequenceid=58, filesize=11.7 K 2024-12-08T21:15:15,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/354ece3f707544eaa5f749749c5763b2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/354ece3f707544eaa5f749749c5763b2 2024-12-08T21:15:15,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/354ece3f707544eaa5f749749c5763b2, entries=150, sequenceid=58, filesize=11.7 K 2024-12-08T21:15:15,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b7d9f8874d392b78c0ba8e541e909290 in 1284ms, sequenceid=58, compaction requested=true 2024-12-08T21:15:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:15,084 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:15,084 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:15,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:15,085 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:15,085 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:15,085 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:15,085 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/47d00e4576724f978b89a60e55ca23e9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/cd991e8624e54e068950add7827501f8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/475222b469c04dd48d3dfc86807d33e8] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=37.4 K 2024-12-08T21:15:15,086 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47d00e4576724f978b89a60e55ca23e9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733692512767 2024-12-08T21:15:15,086 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd991e8624e54e068950add7827501f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733692512831 2024-12-08T21:15:15,086 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 475222b469c04dd48d3dfc86807d33e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733692513133 2024-12-08T21:15:15,091 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:15,091 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:15,091 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:15,091 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ccbddf5acaa244f0b27f56343b64c741, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/b9169cd519a74089aade947246ca846f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/029eeef15ee943429ffae6fab4d1b4e9] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.2 K 2024-12-08T21:15:15,091 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ccbddf5acaa244f0b27f56343b64c741, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733692512773 2024-12-08T21:15:15,092 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b9169cd519a74089aade947246ca846f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733692512831 2024-12-08T21:15:15,092 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 029eeef15ee943429ffae6fab4d1b4e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733692513133 2024-12-08T21:15:15,096 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#194 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:15,097 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c6b417d43b264648bc08c6d7910ae646 is 50, key is test_row_0/A:col10/1733692513169/Put/seqid=0 2024-12-08T21:15:15,100 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:15,101 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/15f3ff0faff0416badabbd7f8c2ab8c6 is 50, key is test_row_0/B:col10/1733692513169/Put/seqid=0 2024-12-08T21:15:15,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742057_1233 (size=12104) 2024-12-08T21:15:15,113 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/15f3ff0faff0416badabbd7f8c2ab8c6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/15f3ff0faff0416badabbd7f8c2ab8c6 2024-12-08T21:15:15,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742058_1234 (size=12104) 2024-12-08T21:15:15,120 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into 15f3ff0faff0416badabbd7f8c2ab8c6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:15,120 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:15,121 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=13, startTime=1733692515084; duration=0sec 2024-12-08T21:15:15,121 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:15,121 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:15,121 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:15,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-08T21:15:15,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:15,124 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:15:15,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:15,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:15,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:15,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:15,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:15,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:15,127 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c6b417d43b264648bc08c6d7910ae646 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c6b417d43b264648bc08c6d7910ae646 2024-12-08T21:15:15,127 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:15,127 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:15,127 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:15,128 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/cd0f60d0f95e4cc5afccaa8dd1e27316, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/151aee0698414259bc31d0af3ae8e078, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/354ece3f707544eaa5f749749c5763b2] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.2 K 2024-12-08T21:15:15,129 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting cd0f60d0f95e4cc5afccaa8dd1e27316, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733692512773 2024-12-08T21:15:15,129 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 151aee0698414259bc31d0af3ae8e078, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733692512831 2024-12-08T21:15:15,131 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 354ece3f707544eaa5f749749c5763b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733692513133 2024-12-08T21:15:15,134 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into c6b417d43b264648bc08c6d7910ae646(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:15,135 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:15,135 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=13, startTime=1733692515084; duration=0sec 2024-12-08T21:15:15,135 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:15,135 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:15,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/7c7211558d9041b584088ec55a340ca2 is 50, key is test_row_0/A:col10/1733692513830/Put/seqid=0 2024-12-08T21:15:15,158 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:15,159 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7b2c17f30f9b4fc6ae1a5a87569f011a is 50, key is test_row_0/C:col10/1733692513169/Put/seqid=0 2024-12-08T21:15:15,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742060_1236 (size=12104) 2024-12-08T21:15:15,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742059_1235 (size=12001) 2024-12-08T21:15:15,573 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/7c7211558d9041b584088ec55a340ca2 2024-12-08T21:15:15,573 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7b2c17f30f9b4fc6ae1a5a87569f011a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2c17f30f9b4fc6ae1a5a87569f011a 2024-12-08T21:15:15,577 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into 7b2c17f30f9b4fc6ae1a5a87569f011a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:15,577 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:15,577 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=13, startTime=1733692515084; duration=0sec 2024-12-08T21:15:15,578 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:15,578 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:15,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/184d38cbbf7a4614bf8119be54f26640 is 50, key is test_row_0/B:col10/1733692513830/Put/seqid=0 2024-12-08T21:15:15,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742061_1237 (size=12001) 2024-12-08T21:15:15,705 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:15:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:15,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:15,984 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/184d38cbbf7a4614bf8119be54f26640 2024-12-08T21:15:15,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/e65098322e40417c8ec8d55df9d9759a is 50, key is test_row_0/C:col10/1733692513830/Put/seqid=0 2024-12-08T21:15:16,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692576002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692576003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742062_1238 (size=12001) 2024-12-08T21:15:16,019 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/e65098322e40417c8ec8d55df9d9759a 2024-12-08T21:15:16,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/7c7211558d9041b584088ec55a340ca2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/7c7211558d9041b584088ec55a340ca2 2024-12-08T21:15:16,031 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/7c7211558d9041b584088ec55a340ca2, entries=150, sequenceid=79, filesize=11.7 K 2024-12-08T21:15:16,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/184d38cbbf7a4614bf8119be54f26640 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/184d38cbbf7a4614bf8119be54f26640 2024-12-08T21:15:16,039 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/184d38cbbf7a4614bf8119be54f26640, entries=150, sequenceid=79, filesize=11.7 K 2024-12-08T21:15:16,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/e65098322e40417c8ec8d55df9d9759a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/e65098322e40417c8ec8d55df9d9759a 2024-12-08T21:15:16,047 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/e65098322e40417c8ec8d55df9d9759a, entries=150, sequenceid=79, filesize=11.7 K 2024-12-08T21:15:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T21:15:16,048 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b7d9f8874d392b78c0ba8e541e909290 in 924ms, sequenceid=79, compaction requested=false 2024-12-08T21:15:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-08T21:15:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-08T21:15:16,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-08T21:15:16,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1490 sec 2024-12-08T21:15:16,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.1560 sec 2024-12-08T21:15:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:16,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:15:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:16,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:16,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:16,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:16,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3781eca2fc3e43ebb83c2e8854f42a28 is 50, key is test_row_0/A:col10/1733692516119/Put/seqid=0 2024-12-08T21:15:16,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742063_1239 (size=12001) 2024-12-08T21:15:16,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692576157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692576157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692576268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692576268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692576477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692576477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3781eca2fc3e43ebb83c2e8854f42a28 2024-12-08T21:15:16,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d is 50, key is test_row_0/B:col10/1733692516119/Put/seqid=0 2024-12-08T21:15:16,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742064_1240 (size=12001) 2024-12-08T21:15:16,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d 2024-12-08T21:15:16,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/a813fbbeb4a84024a7a1ba034d8feb88 is 50, key is test_row_0/C:col10/1733692516119/Put/seqid=0 2024-12-08T21:15:16,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742065_1241 (size=12001) 2024-12-08T21:15:16,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/a813fbbeb4a84024a7a1ba034d8feb88 2024-12-08T21:15:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3781eca2fc3e43ebb83c2e8854f42a28 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3781eca2fc3e43ebb83c2e8854f42a28 2024-12-08T21:15:16,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3781eca2fc3e43ebb83c2e8854f42a28, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T21:15:16,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d 2024-12-08T21:15:16,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T21:15:16,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/a813fbbeb4a84024a7a1ba034d8feb88 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a813fbbeb4a84024a7a1ba034d8feb88 2024-12-08T21:15:16,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a813fbbeb4a84024a7a1ba034d8feb88, entries=150, sequenceid=95, filesize=11.7 K 2024-12-08T21:15:16,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for b7d9f8874d392b78c0ba8e541e909290 in 474ms, sequenceid=95, compaction requested=true 2024-12-08T21:15:16,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:16,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:16,593 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:16,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:16,593 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:16,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:16,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:16,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:16,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:16,594 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:16,594 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:16,594 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:16,594 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:16,594 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:16,594 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:16,595 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c6b417d43b264648bc08c6d7910ae646, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/7c7211558d9041b584088ec55a340ca2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3781eca2fc3e43ebb83c2e8854f42a28] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.3 K 2024-12-08T21:15:16,595 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/15f3ff0faff0416badabbd7f8c2ab8c6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/184d38cbbf7a4614bf8119be54f26640, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.3 K 2024-12-08T21:15:16,595 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b417d43b264648bc08c6d7910ae646, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733692513133 2024-12-08T21:15:16,595 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 15f3ff0faff0416badabbd7f8c2ab8c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733692513133 2024-12-08T21:15:16,595 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c7211558d9041b584088ec55a340ca2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733692513824 2024-12-08T21:15:16,595 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 184d38cbbf7a4614bf8119be54f26640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733692513824 2024-12-08T21:15:16,596 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a54ae96e8cf4ebcb7d14df5a7c3b22d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733692516000 2024-12-08T21:15:16,596 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3781eca2fc3e43ebb83c2e8854f42a28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733692516000 2024-12-08T21:15:16,605 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:16,605 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/a5d6e0a1bbbe4577829f53fe7a00f74b is 50, key is test_row_0/B:col10/1733692516119/Put/seqid=0 2024-12-08T21:15:16,611 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:16,612 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/64e3136f341a4f34a1794106411afd75 is 50, key is test_row_0/A:col10/1733692516119/Put/seqid=0 2024-12-08T21:15:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742066_1242 (size=12207) 2024-12-08T21:15:16,634 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/a5d6e0a1bbbe4577829f53fe7a00f74b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a5d6e0a1bbbe4577829f53fe7a00f74b 2024-12-08T21:15:16,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742067_1243 (size=12207) 2024-12-08T21:15:16,643 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into a5d6e0a1bbbe4577829f53fe7a00f74b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:16,643 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:16,643 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=13, startTime=1733692516593; duration=0sec 2024-12-08T21:15:16,643 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:16,643 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:16,643 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:16,645 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:16,645 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:16,645 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:16,645 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2c17f30f9b4fc6ae1a5a87569f011a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/e65098322e40417c8ec8d55df9d9759a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a813fbbeb4a84024a7a1ba034d8feb88] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.3 K 2024-12-08T21:15:16,645 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b2c17f30f9b4fc6ae1a5a87569f011a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733692513133 2024-12-08T21:15:16,646 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e65098322e40417c8ec8d55df9d9759a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733692513824 2024-12-08T21:15:16,646 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a813fbbeb4a84024a7a1ba034d8feb88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733692516000 2024-12-08T21:15:16,648 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/64e3136f341a4f34a1794106411afd75 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/64e3136f341a4f34a1794106411afd75 2024-12-08T21:15:16,657 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 64e3136f341a4f34a1794106411afd75(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:16,657 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:16,657 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=13, startTime=1733692516593; duration=0sec 2024-12-08T21:15:16,657 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:16,657 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:16,658 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:16,658 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7d68953e0acb4966a8a53b91720f8900 is 50, key is test_row_0/C:col10/1733692516119/Put/seqid=0 2024-12-08T21:15:16,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742068_1244 (size=12207) 2024-12-08T21:15:16,676 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7d68953e0acb4966a8a53b91720f8900 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7d68953e0acb4966a8a53b91720f8900 2024-12-08T21:15:16,682 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into 7d68953e0acb4966a8a53b91720f8900(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:16,682 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:16,683 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=13, startTime=1733692516594; duration=0sec 2024-12-08T21:15:16,683 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:16,683 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:16,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:16,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T21:15:16,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:16,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:16,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:16,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:16,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:16,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:16,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/87451cfe62ab433180b32654aa947a25 is 50, key is test_row_0/A:col10/1733692516794/Put/seqid=0 2024-12-08T21:15:16,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692576818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692576819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742069_1245 (size=12001) 2024-12-08T21:15:16,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/87451cfe62ab433180b32654aa947a25 2024-12-08T21:15:16,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/2d245f4a20bd41fbbd7f2c17c354dda9 is 50, key is test_row_0/B:col10/1733692516794/Put/seqid=0 2024-12-08T21:15:16,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742070_1246 (size=12001) 2024-12-08T21:15:16,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/2d245f4a20bd41fbbd7f2c17c354dda9 2024-12-08T21:15:16,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7f7b12219e1c459d904ddaddd800b7c7 is 50, key is test_row_0/C:col10/1733692516794/Put/seqid=0 2024-12-08T21:15:16,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742071_1247 (size=12001) 2024-12-08T21:15:16,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692576928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692576928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:16,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692577018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,020 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692577030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,031 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692577038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,040 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692577138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692577138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7f7b12219e1c459d904ddaddd800b7c7 2024-12-08T21:15:17,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/87451cfe62ab433180b32654aa947a25 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/87451cfe62ab433180b32654aa947a25 2024-12-08T21:15:17,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/87451cfe62ab433180b32654aa947a25, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T21:15:17,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/2d245f4a20bd41fbbd7f2c17c354dda9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d245f4a20bd41fbbd7f2c17c354dda9 2024-12-08T21:15:17,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d245f4a20bd41fbbd7f2c17c354dda9, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T21:15:17,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7f7b12219e1c459d904ddaddd800b7c7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7f7b12219e1c459d904ddaddd800b7c7 2024-12-08T21:15:17,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7f7b12219e1c459d904ddaddd800b7c7, entries=150, sequenceid=118, filesize=11.7 K 2024-12-08T21:15:17,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b7d9f8874d392b78c0ba8e541e909290 in 523ms, sequenceid=118, compaction requested=false 2024-12-08T21:15:17,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:17,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:17,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:15:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:17,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:17,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/66cf2aee4a1c40d981903a98ad5d9a14 is 50, key is test_row_0/A:col10/1733692516817/Put/seqid=0 2024-12-08T21:15:17,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742072_1248 (size=12101) 2024-12-08T21:15:17,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692577484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692577485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692577598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692577598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692577808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692577808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:17,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:17,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/66cf2aee4a1c40d981903a98ad5d9a14 2024-12-08T21:15:17,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d6b62c57ed1b4acdb1d8397e7dec3097 is 50, key is test_row_0/B:col10/1733692516817/Put/seqid=0 2024-12-08T21:15:17,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742073_1249 (size=12101) 2024-12-08T21:15:17,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d6b62c57ed1b4acdb1d8397e7dec3097 2024-12-08T21:15:17,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/74921ab3c28f43d0bd98d203584423f1 is 50, key is test_row_0/C:col10/1733692516817/Put/seqid=0 2024-12-08T21:15:17,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742074_1250 (size=12101) 2024-12-08T21:15:17,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/74921ab3c28f43d0bd98d203584423f1 2024-12-08T21:15:17,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/66cf2aee4a1c40d981903a98ad5d9a14 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/66cf2aee4a1c40d981903a98ad5d9a14 2024-12-08T21:15:17,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/66cf2aee4a1c40d981903a98ad5d9a14, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T21:15:17,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d6b62c57ed1b4acdb1d8397e7dec3097 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d6b62c57ed1b4acdb1d8397e7dec3097 2024-12-08T21:15:17,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d6b62c57ed1b4acdb1d8397e7dec3097, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T21:15:17,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/74921ab3c28f43d0bd98d203584423f1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/74921ab3c28f43d0bd98d203584423f1 2024-12-08T21:15:17,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/74921ab3c28f43d0bd98d203584423f1, entries=150, sequenceid=131, filesize=11.8 K 2024-12-08T21:15:17,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b7d9f8874d392b78c0ba8e541e909290 in 476ms, sequenceid=131, compaction requested=true 2024-12-08T21:15:17,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:17,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:17,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:17,927 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:17,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:17,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:17,927 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:17,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:17,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:17,929 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:17,929 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:17,929 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:17,929 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:17,929 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:17,929 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:17,929 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/64e3136f341a4f34a1794106411afd75, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/87451cfe62ab433180b32654aa947a25, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/66cf2aee4a1c40d981903a98ad5d9a14] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.5 K 2024-12-08T21:15:17,929 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a5d6e0a1bbbe4577829f53fe7a00f74b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d245f4a20bd41fbbd7f2c17c354dda9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d6b62c57ed1b4acdb1d8397e7dec3097] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.5 K 2024-12-08T21:15:17,930 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a5d6e0a1bbbe4577829f53fe7a00f74b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733692516000 2024-12-08T21:15:17,930 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e3136f341a4f34a1794106411afd75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733692516000 2024-12-08T21:15:17,930 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d245f4a20bd41fbbd7f2c17c354dda9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733692516150 2024-12-08T21:15:17,930 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87451cfe62ab433180b32654aa947a25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733692516150 2024-12-08T21:15:17,930 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b62c57ed1b4acdb1d8397e7dec3097, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733692516805 2024-12-08T21:15:17,930 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66cf2aee4a1c40d981903a98ad5d9a14, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733692516805 2024-12-08T21:15:17,951 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:17,951 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/343ab9d9485c4617af7be421b8ede725 is 50, key is test_row_0/B:col10/1733692516817/Put/seqid=0 2024-12-08T21:15:17,963 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:17,963 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/737ec47afc374157b3bf0d280cc7b3ce is 50, key is test_row_0/A:col10/1733692516817/Put/seqid=0 2024-12-08T21:15:17,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742076_1252 (size=12409) 2024-12-08T21:15:17,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742075_1251 (size=12409) 2024-12-08T21:15:17,981 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/737ec47afc374157b3bf0d280cc7b3ce as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/737ec47afc374157b3bf0d280cc7b3ce 2024-12-08T21:15:17,984 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/343ab9d9485c4617af7be421b8ede725 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/343ab9d9485c4617af7be421b8ede725 2024-12-08T21:15:17,987 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 737ec47afc374157b3bf0d280cc7b3ce(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:17,987 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:17,987 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=13, startTime=1733692517927; duration=0sec 2024-12-08T21:15:17,987 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:17,987 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:17,987 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:17,988 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:17,988 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:17,988 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:17,988 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7d68953e0acb4966a8a53b91720f8900, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7f7b12219e1c459d904ddaddd800b7c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/74921ab3c28f43d0bd98d203584423f1] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=35.5 K 2024-12-08T21:15:17,989 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d68953e0acb4966a8a53b91720f8900, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733692516000 2024-12-08T21:15:17,990 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7b12219e1c459d904ddaddd800b7c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733692516150 2024-12-08T21:15:17,990 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into 343ab9d9485c4617af7be421b8ede725(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:17,990 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:17,990 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=13, startTime=1733692517927; duration=0sec 2024-12-08T21:15:17,990 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74921ab3c28f43d0bd98d203584423f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733692516805 2024-12-08T21:15:17,990 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:17,990 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:17,996 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:17,997 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/d45878a59ccf40679c0bd3e45214c94c is 50, key is test_row_0/C:col10/1733692516817/Put/seqid=0 2024-12-08T21:15:18,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742077_1253 (size=12409) 2024-12-08T21:15:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-08T21:15:18,068 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:18,068 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:18,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-08T21:15:18,070 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T21:15:18,071 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:18,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:18,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:15:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:18,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:18,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/70a570d16ce144b3834e08f8f947bbb9 is 50, key is test_row_0/A:col10/1733692517480/Put/seqid=0 2024-12-08T21:15:18,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742078_1254 (size=12151) 2024-12-08T21:15:18,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692578137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692578138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T21:15:18,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T21:15:18,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:18,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692578248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692578248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T21:15:18,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T21:15:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:18,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,416 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/d45878a59ccf40679c0bd3e45214c94c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d45878a59ccf40679c0bd3e45214c94c 2024-12-08T21:15:18,421 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into d45878a59ccf40679c0bd3e45214c94c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:18,421 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:18,421 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=13, startTime=1733692517927; duration=0sec 2024-12-08T21:15:18,421 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:18,421 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692578457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692578457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/70a570d16ce144b3834e08f8f947bbb9 2024-12-08T21:15:18,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T21:15:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:18,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/f939440f332d4cc08042801aacba8b0d is 50, key is test_row_0/B:col10/1733692517480/Put/seqid=0 2024-12-08T21:15:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:18,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:18,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742079_1255 (size=12151) 2024-12-08T21:15:18,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/f939440f332d4cc08042801aacba8b0d 2024-12-08T21:15:18,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/917f31595e7745f89e5ce4562ea25f9d is 50, key is test_row_0/C:col10/1733692517480/Put/seqid=0 2024-12-08T21:15:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742080_1256 (size=12151) 2024-12-08T21:15:18,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/917f31595e7745f89e5ce4562ea25f9d 2024-12-08T21:15:18,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/70a570d16ce144b3834e08f8f947bbb9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/70a570d16ce144b3834e08f8f947bbb9 2024-12-08T21:15:18,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/70a570d16ce144b3834e08f8f947bbb9, entries=150, sequenceid=153, filesize=11.9 K 2024-12-08T21:15:18,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/f939440f332d4cc08042801aacba8b0d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/f939440f332d4cc08042801aacba8b0d 2024-12-08T21:15:18,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/f939440f332d4cc08042801aacba8b0d, entries=150, sequenceid=153, filesize=11.9 K 2024-12-08T21:15:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/917f31595e7745f89e5ce4562ea25f9d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/917f31595e7745f89e5ce4562ea25f9d 2024-12-08T21:15:18,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/917f31595e7745f89e5ce4562ea25f9d, entries=150, sequenceid=153, filesize=11.9 K 2024-12-08T21:15:18,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for b7d9f8874d392b78c0ba8e541e909290 in 476ms, sequenceid=153, compaction requested=false 2024-12-08T21:15:18,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:18,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-08T21:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:18,695 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:18,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:18,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T21:15:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/b82dec05c6d7412bb6b7b4789609b6ab is 50, key is test_row_0/A:col10/1733692518123/Put/seqid=0 2024-12-08T21:15:18,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742081_1257 (size=12151) 2024-12-08T21:15:18,713 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/b82dec05c6d7412bb6b7b4789609b6ab 2024-12-08T21:15:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d49f368a7e3c4822a88d9806a6dd5bfd is 50, key is test_row_0/B:col10/1733692518123/Put/seqid=0 2024-12-08T21:15:18,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742082_1258 (size=12151) 2024-12-08T21:15:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:18,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692518795, max timestamp=1733692518805Num uniques 859; 2024-12-08T21:15:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:15:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692518795, max timestamp=1733692518805Num uniques 859; 2024-12-08T21:15:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:15:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692518795, max timestamp=1733692518805Num uniques 859; 2024-12-08T21:15:18,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:15:18,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692578832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692578835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692578938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692578947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d49f368a7e3c4822a88d9806a6dd5bfd 2024-12-08T21:15:19,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/d0d08b400d4c4698a7ec30e0cf2158eb is 50, key is test_row_0/C:col10/1733692518123/Put/seqid=0 2024-12-08T21:15:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692579148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692579158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742083_1259 (size=12151) 2024-12-08T21:15:19,168 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/d0d08b400d4c4698a7ec30e0cf2158eb 2024-12-08T21:15:19,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/b82dec05c6d7412bb6b7b4789609b6ab as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/b82dec05c6d7412bb6b7b4789609b6ab 2024-12-08T21:15:19,183 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/b82dec05c6d7412bb6b7b4789609b6ab, entries=150, sequenceid=166, filesize=11.9 K 2024-12-08T21:15:19,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d49f368a7e3c4822a88d9806a6dd5bfd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d49f368a7e3c4822a88d9806a6dd5bfd 2024-12-08T21:15:19,187 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d49f368a7e3c4822a88d9806a6dd5bfd, entries=150, sequenceid=166, filesize=11.9 K 2024-12-08T21:15:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/d0d08b400d4c4698a7ec30e0cf2158eb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0d08b400d4c4698a7ec30e0cf2158eb 2024-12-08T21:15:19,192 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0d08b400d4c4698a7ec30e0cf2158eb, entries=150, sequenceid=166, filesize=11.9 K 2024-12-08T21:15:19,193 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for b7d9f8874d392b78c0ba8e541e909290 in 497ms, sequenceid=166, compaction requested=true 2024-12-08T21:15:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-08T21:15:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-08T21:15:19,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-08T21:15:19,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1260 sec 2024-12-08T21:15:19,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.1310 sec 2024-12-08T21:15:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-08T21:15:19,208 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:19,208 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-08T21:15:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T21:15:19,211 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:19,212 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:19,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T21:15:19,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-08T21:15:19,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:19,364 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-08T21:15:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:19,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:19,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/35b6a6bc20d440828cb1e243cd3090bb is 50, key is test_row_0/A:col10/1733692518831/Put/seqid=0 2024-12-08T21:15:19,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742084_1260 (size=12151) 2024-12-08T21:15:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:19,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:19,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692579484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692579488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T21:15:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692579598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692579598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,785 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/35b6a6bc20d440828cb1e243cd3090bb 2024-12-08T21:15:19,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/4a983edd64ee4285b68cceaa2d6cee5b is 50, key is test_row_0/B:col10/1733692518831/Put/seqid=0 2024-12-08T21:15:19,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742085_1261 (size=12151) 2024-12-08T21:15:19,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/4a983edd64ee4285b68cceaa2d6cee5b 2024-12-08T21:15:19,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692579808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:19,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692579808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:19,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/19d6e9efa8634debbc177c35c8701fef is 50, key is test_row_0/C:col10/1733692518831/Put/seqid=0 2024-12-08T21:15:19,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742086_1262 (size=12151) 2024-12-08T21:15:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T21:15:20,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692580117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692580118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/19d6e9efa8634debbc177c35c8701fef 2024-12-08T21:15:20,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/35b6a6bc20d440828cb1e243cd3090bb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/35b6a6bc20d440828cb1e243cd3090bb 2024-12-08T21:15:20,233 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/35b6a6bc20d440828cb1e243cd3090bb, entries=150, sequenceid=200, filesize=11.9 K 2024-12-08T21:15:20,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/4a983edd64ee4285b68cceaa2d6cee5b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/4a983edd64ee4285b68cceaa2d6cee5b 2024-12-08T21:15:20,238 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/4a983edd64ee4285b68cceaa2d6cee5b, entries=150, sequenceid=200, filesize=11.9 K 2024-12-08T21:15:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/19d6e9efa8634debbc177c35c8701fef as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/19d6e9efa8634debbc177c35c8701fef 2024-12-08T21:15:20,243 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/19d6e9efa8634debbc177c35c8701fef, entries=150, sequenceid=200, filesize=11.9 K 2024-12-08T21:15:20,244 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for b7d9f8874d392b78c0ba8e541e909290 in 880ms, sequenceid=200, compaction requested=true 2024-12-08T21:15:20,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:20,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:20,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-08T21:15:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-08T21:15:20,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-08T21:15:20,247 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-12-08T21:15:20,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0390 sec 2024-12-08T21:15:20,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-08T21:15:20,348 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:20,349 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-08T21:15:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T21:15:20,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:20,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:20,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T21:15:20,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-08T21:15:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:20,504 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:15:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:20,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/ed846df795e14be39df054b9da1de0e4 is 50, key is test_row_0/A:col10/1733692519481/Put/seqid=0 2024-12-08T21:15:20,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742087_1263 (size=12151) 2024-12-08T21:15:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:20,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:20,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692580663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T21:15:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692580665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692580778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692580778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/ed846df795e14be39df054b9da1de0e4 2024-12-08T21:15:20,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/5ad40511c7fa4ef681ac4ea6c02e39db is 50, key is test_row_0/B:col10/1733692519481/Put/seqid=0 2024-12-08T21:15:20,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742088_1264 (size=12151) 2024-12-08T21:15:20,927 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/5ad40511c7fa4ef681ac4ea6c02e39db 2024-12-08T21:15:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/a3af63ba7f2946248c6bfeec482807f6 is 50, key is test_row_0/C:col10/1733692519481/Put/seqid=0 2024-12-08T21:15:20,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742089_1265 (size=12151) 2024-12-08T21:15:20,958 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/a3af63ba7f2946248c6bfeec482807f6 2024-12-08T21:15:20,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/ed846df795e14be39df054b9da1de0e4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/ed846df795e14be39df054b9da1de0e4 2024-12-08T21:15:20,976 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/ed846df795e14be39df054b9da1de0e4, entries=150, sequenceid=212, filesize=11.9 K 2024-12-08T21:15:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/5ad40511c7fa4ef681ac4ea6c02e39db as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5ad40511c7fa4ef681ac4ea6c02e39db 2024-12-08T21:15:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T21:15:20,984 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5ad40511c7fa4ef681ac4ea6c02e39db, entries=150, sequenceid=212, filesize=11.9 K 2024-12-08T21:15:20,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/a3af63ba7f2946248c6bfeec482807f6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a3af63ba7f2946248c6bfeec482807f6 2024-12-08T21:15:20,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692580987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692580988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:20,995 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a3af63ba7f2946248c6bfeec482807f6, entries=150, sequenceid=212, filesize=11.9 K 2024-12-08T21:15:20,995 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b7d9f8874d392b78c0ba8e541e909290 in 491ms, sequenceid=212, compaction requested=true 2024-12-08T21:15:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-08T21:15:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-08T21:15:20,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-08T21:15:20,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 645 msec 2024-12-08T21:15:21,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 650 msec 2024-12-08T21:15:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:21,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T21:15:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:21,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692520657, max timestamp=1733692521060Num uniques 859; 2024-12-08T21:15:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:21,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:21,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:21,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/353ee7baf18d46aaa56a27d863c9343b is 50, key is test_row_0/A:col10/1733692521062/Put/seqid=0 2024-12-08T21:15:21,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742090_1266 (size=16931) 2024-12-08T21:15:21,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692581083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692581087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692581088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692581198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692581198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692581198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692581298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692581298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692581407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692581407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692581418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/353ee7baf18d46aaa56a27d863c9343b 2024-12-08T21:15:21,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/bed89cf0b89447b2ab7faa06778dded9 is 50, key is test_row_0/B:col10/1733692521062/Put/seqid=0 2024-12-08T21:15:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-08T21:15:21,488 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:21,489 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-08T21:15:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742091_1267 (size=12151) 2024-12-08T21:15:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T21:15:21,490 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:21,491 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:21,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T21:15:21,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T21:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:21,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692581718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692581718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692581728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T21:15:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:21,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T21:15:21,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692581808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692581808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:21,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/bed89cf0b89447b2ab7faa06778dded9 2024-12-08T21:15:21,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/891dca7c3d8d4965abe3193d18116e68 is 50, key is test_row_0/C:col10/1733692521062/Put/seqid=0 2024-12-08T21:15:21,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742092_1268 (size=12151) 2024-12-08T21:15:21,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T21:15:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:21,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:22,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T21:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T21:15:22,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692582227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692582227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:22,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692582238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:22,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T21:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:22,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/891dca7c3d8d4965abe3193d18116e68 2024-12-08T21:15:22,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/353ee7baf18d46aaa56a27d863c9343b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/353ee7baf18d46aaa56a27d863c9343b 2024-12-08T21:15:22,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/353ee7baf18d46aaa56a27d863c9343b, entries=250, sequenceid=234, filesize=16.5 K 2024-12-08T21:15:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/bed89cf0b89447b2ab7faa06778dded9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bed89cf0b89447b2ab7faa06778dded9 2024-12-08T21:15:22,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bed89cf0b89447b2ab7faa06778dded9, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T21:15:22,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/891dca7c3d8d4965abe3193d18116e68 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/891dca7c3d8d4965abe3193d18116e68 2024-12-08T21:15:22,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/891dca7c3d8d4965abe3193d18116e68, entries=150, sequenceid=234, filesize=11.9 K 2024-12-08T21:15:22,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=87.22 KB/89310 for b7d9f8874d392b78c0ba8e541e909290 in 1258ms, sequenceid=234, compaction requested=true 2024-12-08T21:15:22,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:22,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:22,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:22,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:22,320 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-08T21:15:22,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:22,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:22,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T21:15:22,321 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-08T21:15:22,322 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 77944 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-08T21:15:22,322 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:22,322 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,322 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/737ec47afc374157b3bf0d280cc7b3ce, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/70a570d16ce144b3834e08f8f947bbb9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/b82dec05c6d7412bb6b7b4789609b6ab, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/35b6a6bc20d440828cb1e243cd3090bb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/ed846df795e14be39df054b9da1de0e4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/353ee7baf18d46aaa56a27d863c9343b] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=76.1 K 2024-12-08T21:15:22,323 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73164 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-08T21:15:22,323 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:22,323 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,323 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/343ab9d9485c4617af7be421b8ede725, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/f939440f332d4cc08042801aacba8b0d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d49f368a7e3c4822a88d9806a6dd5bfd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/4a983edd64ee4285b68cceaa2d6cee5b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5ad40511c7fa4ef681ac4ea6c02e39db, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bed89cf0b89447b2ab7faa06778dded9] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=71.4 K 2024-12-08T21:15:22,324 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 343ab9d9485c4617af7be421b8ede725, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733692516805 2024-12-08T21:15:22,324 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737ec47afc374157b3bf0d280cc7b3ce, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733692516805 2024-12-08T21:15:22,324 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f939440f332d4cc08042801aacba8b0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733692517477 2024-12-08T21:15:22,324 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70a570d16ce144b3834e08f8f947bbb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733692517477 2024-12-08T21:15:22,324 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b82dec05c6d7412bb6b7b4789609b6ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733692518123 2024-12-08T21:15:22,324 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting d49f368a7e3c4822a88d9806a6dd5bfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733692518123 2024-12-08T21:15:22,325 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a983edd64ee4285b68cceaa2d6cee5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692518830 2024-12-08T21:15:22,325 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35b6a6bc20d440828cb1e243cd3090bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692518830 2024-12-08T21:15:22,325 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed846df795e14be39df054b9da1de0e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733692519476 2024-12-08T21:15:22,325 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ad40511c7fa4ef681ac4ea6c02e39db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733692519476 2024-12-08T21:15:22,325 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting bed89cf0b89447b2ab7faa06778dded9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733692520657 2024-12-08T21:15:22,325 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353ee7baf18d46aaa56a27d863c9343b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733692520657 2024-12-08T21:15:22,341 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#230 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:22,341 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/761ecf5b23ac4a918e60e4bde154de07 is 50, key is test_row_0/A:col10/1733692521062/Put/seqid=0 2024-12-08T21:15:22,344 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#231 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:22,344 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d04c07257a254b548be87c7607d1a502 is 50, key is test_row_0/B:col10/1733692521062/Put/seqid=0 2024-12-08T21:15:22,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742093_1269 (size=12663) 2024-12-08T21:15:22,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742094_1270 (size=12663) 2024-12-08T21:15:22,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-08T21:15:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,408 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T21:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:22,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/0664e2b4c10c4c1aaad1c4f15ec534b4 is 50, key is test_row_0/A:col10/1733692521085/Put/seqid=0 2024-12-08T21:15:22,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742095_1271 (size=12151) 2024-12-08T21:15:22,416 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/0664e2b4c10c4c1aaad1c4f15ec534b4 2024-12-08T21:15:22,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7eaf840fe0994790ab24f8cdbf688976 is 50, key is test_row_0/B:col10/1733692521085/Put/seqid=0 2024-12-08T21:15:22,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742096_1272 (size=12151) 2024-12-08T21:15:22,430 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7eaf840fe0994790ab24f8cdbf688976 2024-12-08T21:15:22,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/05779387652348d2be3bc5a2d1554488 is 50, key is test_row_0/C:col10/1733692521085/Put/seqid=0 2024-12-08T21:15:22,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742097_1273 (size=12151) 2024-12-08T21:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T21:15:22,757 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/761ecf5b23ac4a918e60e4bde154de07 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/761ecf5b23ac4a918e60e4bde154de07 2024-12-08T21:15:22,761 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 761ecf5b23ac4a918e60e4bde154de07(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:22,761 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:22,762 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=10, startTime=1733692522320; duration=0sec 2024-12-08T21:15:22,762 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:22,762 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:22,762 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-08T21:15:22,762 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d04c07257a254b548be87c7607d1a502 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d04c07257a254b548be87c7607d1a502 2024-12-08T21:15:22,765 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73164 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-08T21:15:22,767 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:22,767 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,767 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d45878a59ccf40679c0bd3e45214c94c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/917f31595e7745f89e5ce4562ea25f9d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0d08b400d4c4698a7ec30e0cf2158eb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/19d6e9efa8634debbc177c35c8701fef, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a3af63ba7f2946248c6bfeec482807f6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/891dca7c3d8d4965abe3193d18116e68] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=71.4 K 2024-12-08T21:15:22,768 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting d45878a59ccf40679c0bd3e45214c94c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733692516805 2024-12-08T21:15:22,769 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 917f31595e7745f89e5ce4562ea25f9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733692517477 2024-12-08T21:15:22,769 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d08b400d4c4698a7ec30e0cf2158eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733692518123 2024-12-08T21:15:22,769 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into d04c07257a254b548be87c7607d1a502(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:22,769 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:22,769 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=10, startTime=1733692522320; duration=0sec 2024-12-08T21:15:22,769 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:22,769 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:22,769 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d6e9efa8634debbc177c35c8701fef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692518830 2024-12-08T21:15:22,770 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3af63ba7f2946248c6bfeec482807f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1733692519476 2024-12-08T21:15:22,770 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 891dca7c3d8d4965abe3193d18116e68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733692520657 2024-12-08T21:15:22,785 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:22,786 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/82f6c1bdf6cf494e86e6041933c5db87 is 50, key is test_row_0/C:col10/1733692521062/Put/seqid=0 2024-12-08T21:15:22,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742098_1274 (size=12663) 2024-12-08T21:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:22,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692582843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:22,846 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/05779387652348d2be3bc5a2d1554488 2024-12-08T21:15:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:22,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692582845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:22,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:22,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/0664e2b4c10c4c1aaad1c4f15ec534b4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0664e2b4c10c4c1aaad1c4f15ec534b4 2024-12-08T21:15:22,855 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0664e2b4c10c4c1aaad1c4f15ec534b4, entries=150, sequenceid=250, filesize=11.9 K 2024-12-08T21:15:22,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7eaf840fe0994790ab24f8cdbf688976 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7eaf840fe0994790ab24f8cdbf688976 2024-12-08T21:15:22,861 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7eaf840fe0994790ab24f8cdbf688976, entries=150, sequenceid=250, filesize=11.9 K 2024-12-08T21:15:22,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/05779387652348d2be3bc5a2d1554488 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/05779387652348d2be3bc5a2d1554488 2024-12-08T21:15:22,865 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/05779387652348d2be3bc5a2d1554488, entries=150, sequenceid=250, filesize=11.9 K 2024-12-08T21:15:22,865 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7d9f8874d392b78c0ba8e541e909290 in 457ms, sequenceid=250, compaction requested=false 2024-12-08T21:15:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-08T21:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-08T21:15:22,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-08T21:15:22,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3750 sec 2024-12-08T21:15:22,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.3790 sec 2024-12-08T21:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:22,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T21:15:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:22,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/4217160411ae4253a1581601cec1cc77 is 50, key is test_row_0/A:col10/1733692522959/Put/seqid=0 2024-12-08T21:15:22,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742099_1275 (size=14741) 2024-12-08T21:15:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692582994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:22,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692582994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692583107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692583108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,195 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/82f6c1bdf6cf494e86e6041933c5db87 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/82f6c1bdf6cf494e86e6041933c5db87 2024-12-08T21:15:23,201 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into 82f6c1bdf6cf494e86e6041933c5db87(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:23,201 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:23,201 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=10, startTime=1733692522320; duration=0sec 2024-12-08T21:15:23,201 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:23,201 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:23,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692583238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,240 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692583248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,250 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692583258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,261 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692583318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692583318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/4217160411ae4253a1581601cec1cc77 2024-12-08T21:15:23,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/93acd9396892432b988d9bc027c36261 is 50, key is test_row_0/B:col10/1733692522959/Put/seqid=0 2024-12-08T21:15:23,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742100_1276 (size=12301) 2024-12-08T21:15:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692583628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692583628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:23,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-08T21:15:23,638 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:23,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-08T21:15:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T21:15:23,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:23,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:23,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T21:15:23,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/93acd9396892432b988d9bc027c36261 2024-12-08T21:15:23,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T21:15:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:23,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:23,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:23,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/79df9e842c4f4303b4f30e8bceb560df is 50, key is test_row_0/C:col10/1733692522959/Put/seqid=0 2024-12-08T21:15:23,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742101_1277 (size=12301) 2024-12-08T21:15:23,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T21:15:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:23,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:23,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:23,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T21:15:24,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T21:15:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:24,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:24,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:24,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692584137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:24,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692584138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:24,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:24,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/79df9e842c4f4303b4f30e8bceb560df 2024-12-08T21:15:24,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/4217160411ae4253a1581601cec1cc77 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4217160411ae4253a1581601cec1cc77 2024-12-08T21:15:24,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4217160411ae4253a1581601cec1cc77, entries=200, sequenceid=270, filesize=14.4 K 2024-12-08T21:15:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/93acd9396892432b988d9bc027c36261 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/93acd9396892432b988d9bc027c36261 2024-12-08T21:15:24,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/93acd9396892432b988d9bc027c36261, entries=150, sequenceid=270, filesize=12.0 K 2024-12-08T21:15:24,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/79df9e842c4f4303b4f30e8bceb560df as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/79df9e842c4f4303b4f30e8bceb560df 2024-12-08T21:15:24,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/79df9e842c4f4303b4f30e8bceb560df, entries=150, sequenceid=270, filesize=12.0 K 2024-12-08T21:15:24,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for b7d9f8874d392b78c0ba8e541e909290 in 1285ms, sequenceid=270, compaction requested=true 2024-12-08T21:15:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:24,245 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:24,245 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:24,246 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:24,246 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:24,246 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:24,246 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:24,246 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:24,246 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:24,246 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d04c07257a254b548be87c7607d1a502, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7eaf840fe0994790ab24f8cdbf688976, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/93acd9396892432b988d9bc027c36261] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=36.2 K 2024-12-08T21:15:24,246 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/761ecf5b23ac4a918e60e4bde154de07, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0664e2b4c10c4c1aaad1c4f15ec534b4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4217160411ae4253a1581601cec1cc77] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=38.6 K 2024-12-08T21:15:24,247 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761ecf5b23ac4a918e60e4bde154de07, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733692520657 2024-12-08T21:15:24,247 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting d04c07257a254b548be87c7607d1a502, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733692520657 2024-12-08T21:15:24,247 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0664e2b4c10c4c1aaad1c4f15ec534b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733692521082 2024-12-08T21:15:24,247 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eaf840fe0994790ab24f8cdbf688976, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733692521082 2024-12-08T21:15:24,247 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4217160411ae4253a1581601cec1cc77, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733692522837 2024-12-08T21:15:24,249 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 93acd9396892432b988d9bc027c36261, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733692522844 2024-12-08T21:15:24,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-08T21:15:24,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:24,260 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T21:15:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:24,260 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:24,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:24,261 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/2c5a987a08a64508952979567faa19b5 is 50, key is test_row_0/A:col10/1733692522959/Put/seqid=0 2024-12-08T21:15:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T21:15:24,270 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:24,271 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/bfbd18f8f4234aedb5ccdbe4b11c9d80 is 50, key is test_row_0/B:col10/1733692522959/Put/seqid=0 2024-12-08T21:15:24,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/aa9babfe41774703b92dd9b202ef2e90 is 50, key is test_row_0/A:col10/1733692522987/Put/seqid=0 2024-12-08T21:15:24,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742102_1278 (size=12915) 2024-12-08T21:15:24,309 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/2c5a987a08a64508952979567faa19b5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/2c5a987a08a64508952979567faa19b5 2024-12-08T21:15:24,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742103_1279 (size=12915) 2024-12-08T21:15:24,316 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 2c5a987a08a64508952979567faa19b5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:24,316 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:24,316 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=13, startTime=1733692524245; duration=0sec 2024-12-08T21:15:24,317 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:24,317 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:24,317 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:24,319 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:24,319 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/bfbd18f8f4234aedb5ccdbe4b11c9d80 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bfbd18f8f4234aedb5ccdbe4b11c9d80 2024-12-08T21:15:24,319 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:24,319 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:24,319 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/82f6c1bdf6cf494e86e6041933c5db87, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/05779387652348d2be3bc5a2d1554488, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/79df9e842c4f4303b4f30e8bceb560df] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=36.2 K 2024-12-08T21:15:24,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742104_1280 (size=12301) 2024-12-08T21:15:24,321 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/aa9babfe41774703b92dd9b202ef2e90 2024-12-08T21:15:24,323 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82f6c1bdf6cf494e86e6041933c5db87, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1733692520657 2024-12-08T21:15:24,324 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05779387652348d2be3bc5a2d1554488, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733692521082 2024-12-08T21:15:24,325 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79df9e842c4f4303b4f30e8bceb560df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733692522844 2024-12-08T21:15:24,334 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into bfbd18f8f4234aedb5ccdbe4b11c9d80(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:24,335 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:24,335 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=13, startTime=1733692524245; duration=0sec 2024-12-08T21:15:24,335 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:24,335 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:24,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0589007af3fd4466a50081b21fb9d5b7 is 50, key is test_row_0/B:col10/1733692522987/Put/seqid=0 2024-12-08T21:15:24,349 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:24,350 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/02cb32043be043b5b498b9b66bf50071 is 50, key is test_row_0/C:col10/1733692522959/Put/seqid=0 2024-12-08T21:15:24,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742105_1281 (size=12301) 2024-12-08T21:15:24,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742106_1282 (size=12915) 2024-12-08T21:15:24,774 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0589007af3fd4466a50081b21fb9d5b7 2024-12-08T21:15:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T21:15:24,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/f64bd176c1e145689e302380706ec66c is 50, key is test_row_0/C:col10/1733692522987/Put/seqid=0 2024-12-08T21:15:24,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742107_1283 (size=12301) 2024-12-08T21:15:24,793 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/f64bd176c1e145689e302380706ec66c 2024-12-08T21:15:24,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/aa9babfe41774703b92dd9b202ef2e90 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/aa9babfe41774703b92dd9b202ef2e90 2024-12-08T21:15:24,804 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/02cb32043be043b5b498b9b66bf50071 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/02cb32043be043b5b498b9b66bf50071 2024-12-08T21:15:24,807 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/aa9babfe41774703b92dd9b202ef2e90, entries=150, sequenceid=286, filesize=12.0 K 2024-12-08T21:15:24,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0589007af3fd4466a50081b21fb9d5b7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0589007af3fd4466a50081b21fb9d5b7 2024-12-08T21:15:24,811 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into 02cb32043be043b5b498b9b66bf50071(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:24,811 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:24,811 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=13, startTime=1733692524245; duration=0sec 2024-12-08T21:15:24,811 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:24,811 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:24,817 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0589007af3fd4466a50081b21fb9d5b7, entries=150, sequenceid=286, filesize=12.0 K 2024-12-08T21:15:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/f64bd176c1e145689e302380706ec66c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/f64bd176c1e145689e302380706ec66c 2024-12-08T21:15:24,834 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/f64bd176c1e145689e302380706ec66c, entries=150, sequenceid=286, filesize=12.0 K 2024-12-08T21:15:24,835 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for b7d9f8874d392b78c0ba8e541e909290 in 576ms, sequenceid=286, compaction requested=false 2024-12-08T21:15:24,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:24,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:24,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-08T21:15:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-08T21:15:24,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-08T21:15:24,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1990 sec 2024-12-08T21:15:24,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.2090 sec 2024-12-08T21:15:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:25,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:15:25,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:25,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:25,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:25,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:25,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:25,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/0ceff38d8b04497a9d3c9edc701f0e8e is 50, key is test_row_1/A:col10/1733692525160/Put/seqid=0 2024-12-08T21:15:25,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692525193, max timestamp=1733692525213Num uniques 859; 2024-12-08T21:15:25,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692525193, max timestamp=1733692525213Num uniques 859; 2024-12-08T21:15:25,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:25,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:25,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692525193, max timestamp=1733692525213Num uniques 859; 2024-12-08T21:15:25,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:25,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742108_1284 (size=9857) 2024-12-08T21:15:25,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/0ceff38d8b04497a9d3c9edc701f0e8e 2024-12-08T21:15:25,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/9abf03055e7a430d9812effa35b50654 is 50, key is test_row_1/B:col10/1733692525160/Put/seqid=0 2024-12-08T21:15:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692585258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,263 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:25,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742109_1285 (size=9857) 2024-12-08T21:15:25,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/9abf03055e7a430d9812effa35b50654 2024-12-08T21:15:25,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692585262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692585263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:25,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692585269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,274 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:25,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/3d2eddd71cac46f8addfb3f90776e8aa is 50, key is test_row_1/C:col10/1733692525160/Put/seqid=0 2024-12-08T21:15:25,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692585279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,281 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:25,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742110_1286 (size=9857) 2024-12-08T21:15:25,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/3d2eddd71cac46f8addfb3f90776e8aa 2024-12-08T21:15:25,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/0ceff38d8b04497a9d3c9edc701f0e8e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0ceff38d8b04497a9d3c9edc701f0e8e 2024-12-08T21:15:25,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0ceff38d8b04497a9d3c9edc701f0e8e, entries=100, sequenceid=299, filesize=9.6 K 2024-12-08T21:15:25,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/9abf03055e7a430d9812effa35b50654 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/9abf03055e7a430d9812effa35b50654 2024-12-08T21:15:25,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/9abf03055e7a430d9812effa35b50654, entries=100, sequenceid=299, filesize=9.6 K 2024-12-08T21:15:25,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/3d2eddd71cac46f8addfb3f90776e8aa as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3d2eddd71cac46f8addfb3f90776e8aa 2024-12-08T21:15:25,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3d2eddd71cac46f8addfb3f90776e8aa, entries=100, sequenceid=299, filesize=9.6 K 2024-12-08T21:15:25,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b7d9f8874d392b78c0ba8e541e909290 in 192ms, sequenceid=299, compaction requested=true 2024-12-08T21:15:25,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:25,356 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:25,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:25,357 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:25,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:25,357 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:25,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:25,357 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:25,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:25,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:25,357 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:25,357 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/2c5a987a08a64508952979567faa19b5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/aa9babfe41774703b92dd9b202ef2e90, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0ceff38d8b04497a9d3c9edc701f0e8e] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=34.3 K 2024-12-08T21:15:25,358 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c5a987a08a64508952979567faa19b5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733692522844 2024-12-08T21:15:25,358 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9babfe41774703b92dd9b202ef2e90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692522987 2024-12-08T21:15:25,358 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ceff38d8b04497a9d3c9edc701f0e8e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733692525160 2024-12-08T21:15:25,361 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:25,361 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:25,361 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:25,361 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bfbd18f8f4234aedb5ccdbe4b11c9d80, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0589007af3fd4466a50081b21fb9d5b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/9abf03055e7a430d9812effa35b50654] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=34.3 K 2024-12-08T21:15:25,362 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting bfbd18f8f4234aedb5ccdbe4b11c9d80, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733692522844 2024-12-08T21:15:25,362 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0589007af3fd4466a50081b21fb9d5b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692522987 2024-12-08T21:15:25,363 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 9abf03055e7a430d9812effa35b50654, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733692525160 2024-12-08T21:15:25,373 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:25,374 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:25,374 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/357fac638665462dba64f353280e704e is 50, key is test_row_0/A:col10/1733692522987/Put/seqid=0 2024-12-08T21:15:25,374 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/cdd8c10fb45349afba64902e02508245 is 50, key is test_row_0/B:col10/1733692522987/Put/seqid=0 2024-12-08T21:15:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:25,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692525253, max timestamp=1733692525381Num uniques 1018; 2024-12-08T21:15:25,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T21:15:25,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692525253, max timestamp=1733692525381Num uniques 1018; 2024-12-08T21:15:25,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T21:15:25,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692525253, max timestamp=1733692525381Num uniques 1018; 2024-12-08T21:15:25,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T21:15:25,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-08T21:15:25,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:25,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:25,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:25,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:25,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:25,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:25,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742111_1287 (size=13017) 2024-12-08T21:15:25,410 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/cdd8c10fb45349afba64902e02508245 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/cdd8c10fb45349afba64902e02508245 2024-12-08T21:15:25,419 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into cdd8c10fb45349afba64902e02508245(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:25,420 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:25,420 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=13, startTime=1733692525357; duration=0sec 2024-12-08T21:15:25,422 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:25,422 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:25,423 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742112_1288 (size=13017) 2024-12-08T21:15:25,425 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:25,425 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:25,425 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:25,426 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/02cb32043be043b5b498b9b66bf50071, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/f64bd176c1e145689e302380706ec66c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3d2eddd71cac46f8addfb3f90776e8aa] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=34.3 K 2024-12-08T21:15:25,426 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 02cb32043be043b5b498b9b66bf50071, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1733692522844 2024-12-08T21:15:25,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/8abc48c430b54ae4b7a2337f9fb87ed0 is 50, key is test_row_0/A:col10/1733692525260/Put/seqid=0 2024-12-08T21:15:25,427 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f64bd176c1e145689e302380706ec66c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1733692522987 2024-12-08T21:15:25,428 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2eddd71cac46f8addfb3f90776e8aa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733692525160 2024-12-08T21:15:25,442 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:25,443 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/2faf7e2a801d471091ef5acd9f9027b2 is 50, key is test_row_0/C:col10/1733692522987/Put/seqid=0 2024-12-08T21:15:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742113_1289 (size=12301) 2024-12-08T21:15:25,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/8abc48c430b54ae4b7a2337f9fb87ed0 2024-12-08T21:15:25,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692525434, max timestamp=1733692525460Num uniques 859; 2024-12-08T21:15:25,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:25,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692525434, max timestamp=1733692525460Num uniques 859; 2024-12-08T21:15:25,462 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:25,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742114_1290 (size=13017) 2024-12-08T21:15:25,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733692525434, max timestamp=1733692525460Num uniques 859; 2024-12-08T21:15:25,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-08T21:15:25,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/257e64a765f54043a88d6b5f610aeadd is 50, key is test_row_0/B:col10/1733692525260/Put/seqid=0 2024-12-08T21:15:25,471 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/2faf7e2a801d471091ef5acd9f9027b2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/2faf7e2a801d471091ef5acd9f9027b2 2024-12-08T21:15:25,479 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into 2faf7e2a801d471091ef5acd9f9027b2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:25,479 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:25,480 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=13, startTime=1733692525357; duration=0sec 2024-12-08T21:15:25,480 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:25,480 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:25,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742115_1291 (size=12301) 2024-12-08T21:15:25,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/257e64a765f54043a88d6b5f610aeadd 2024-12-08T21:15:25,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7b2f1275027c463a8d4e67b660aaf92e is 50, key is test_row_0/C:col10/1733692525260/Put/seqid=0 2024-12-08T21:15:25,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742116_1292 (size=12301) 2024-12-08T21:15:25,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692525487, max timestamp=1733692525516Num uniques 1018; 2024-12-08T21:15:25,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T21:15:25,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692525487, max timestamp=1733692525516Num uniques 1018; 2024-12-08T21:15:25,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733692525487, max timestamp=1733692525516Num uniques 1009; 2024-12-08T21:15:25,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-08T21:15:25,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-08T21:15:25,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692525555, max timestamp=1733692525565Num uniques 1027; 2024-12-08T21:15:25,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T21:15:25,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692525555, max timestamp=1733692525565Num uniques 1027; 2024-12-08T21:15:25,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T21:15:25,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692525555, max timestamp=1733692525565Num uniques 1027; 2024-12-08T21:15:25,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-08T21:15:25,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692525596, max timestamp=1733692525604Num uniques 1036; 2024-12-08T21:15:25,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692525596, max timestamp=1733692525604Num uniques 1036; 2024-12-08T21:15:25,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T21:15:25,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T21:15:25,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692525596, max timestamp=1733692525604Num uniques 1036; 2024-12-08T21:15:25,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-08T21:15:25,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692525627, max timestamp=1733692525642Num uniques 1045; 2024-12-08T21:15:25,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-08T21:15:25,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692525627, max timestamp=1733692525642Num uniques 1045; 2024-12-08T21:15:25,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-08T21:15:25,645 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692525627, max timestamp=1733692525642Num uniques 1045; 2024-12-08T21:15:25,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-12-08T21:15:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692585682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733692525674, max timestamp=1733692525685Num uniques 1104; 2024-12-08T21:15:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733692525674, max timestamp=1733692525685Num uniques 1104; 2024-12-08T21:15:25,688 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733692525674, max timestamp=1733692525685Num uniques 1104; 2024-12-08T21:15:25,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T21:15:25,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T21:15:25,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-12-08T21:15:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:25,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733692525731, max timestamp=1733692525749Num uniques 1013; 2024-12-08T21:15:25,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733692525731, max timestamp=1733692525749Num uniques 1013; 2024-12-08T21:15:25,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T21:15:25,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733692525731, max timestamp=1733692525749Num uniques 1013; 2024-12-08T21:15:25,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T21:15:25,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-12-08T21:15:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-08T21:15:25,788 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:25,788 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-08T21:15:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T21:15:25,790 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:25,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:25,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:25,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733692525799, max timestamp=1733692525807Num uniques 1022; 2024-12-08T21:15:25,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-08T21:15:25,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733692525799, max timestamp=1733692525807Num uniques 1022; 2024-12-08T21:15:25,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733692525799, max timestamp=1733692525807Num uniques 1022; 2024-12-08T21:15:25,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-08T21:15:25,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-08T21:15:25,828 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/357fac638665462dba64f353280e704e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/357fac638665462dba64f353280e704e 2024-12-08T21:15:25,833 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 357fac638665462dba64f353280e704e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:25,833 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:25,833 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=13, startTime=1733692525356; duration=0sec 2024-12-08T21:15:25,833 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:25,833 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:25,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733692525846, max timestamp=1733692525853Num uniques 1031; 2024-12-08T21:15:25,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-08T21:15:25,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733692525846, max timestamp=1733692525853Num uniques 1031; 2024-12-08T21:15:25,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-08T21:15:25,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733692525846, max timestamp=1733692525853Num uniques 1031; 2024-12-08T21:15:25,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-08T21:15:25,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7b2f1275027c463a8d4e67b660aaf92e 2024-12-08T21:15:25,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733692525877, max timestamp=1733692525893Num uniques 1040; 2024-12-08T21:15:25,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-08T21:15:25,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733692525877, max timestamp=1733692525893Num uniques 1040; 2024-12-08T21:15:25,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-08T21:15:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T21:15:25,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733692525877, max timestamp=1733692525893Num uniques 1040; 2024-12-08T21:15:25,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-12-08T21:15:25,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/8abc48c430b54ae4b7a2337f9fb87ed0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/8abc48c430b54ae4b7a2337f9fb87ed0 2024-12-08T21:15:25,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/8abc48c430b54ae4b7a2337f9fb87ed0, entries=150, sequenceid=338, filesize=12.0 K 2024-12-08T21:15:25,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/257e64a765f54043a88d6b5f610aeadd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/257e64a765f54043a88d6b5f610aeadd 2024-12-08T21:15:25,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/257e64a765f54043a88d6b5f610aeadd, entries=150, sequenceid=338, filesize=12.0 K 2024-12-08T21:15:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7b2f1275027c463a8d4e67b660aaf92e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2f1275027c463a8d4e67b660aaf92e 2024-12-08T21:15:25,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2f1275027c463a8d4e67b660aaf92e, entries=150, sequenceid=338, filesize=12.0 K 2024-12-08T21:15:25,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=87.22 KB/89310 for b7d9f8874d392b78c0ba8e541e909290 in 533ms, sequenceid=338, compaction requested=false 2024-12-08T21:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:25,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=93.93 KB heapSize=257.86 KB 2024-12-08T21:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:25,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:25,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/4acf26948ea54ff4948a9916a05f595b is 50, key is test_row_0/A:col10/1733692525918/Put/seqid=0 2024-12-08T21:15:25,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742117_1293 (size=14741) 2024-12-08T21:15:25,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:25,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:25,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:25,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:25,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:25,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692585952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692585951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:25,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692586068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:26,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692586068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:26,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:26,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:26,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T21:15:26,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692586278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692586278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:26,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/4acf26948ea54ff4948a9916a05f595b 2024-12-08T21:15:26,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/ee9bd0af6ada4db69c3c1b5aead7502f is 50, key is test_row_0/B:col10/1733692525918/Put/seqid=0 2024-12-08T21:15:26,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742118_1294 (size=12301) 2024-12-08T21:15:26,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T21:15:26,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692586587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:26,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692586588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:26,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:26,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:26,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:26,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/ee9bd0af6ada4db69c3c1b5aead7502f 2024-12-08T21:15:26,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/5a13afb3f70f461eb98311040b045082 is 50, key is test_row_0/C:col10/1733692525918/Put/seqid=0 2024-12-08T21:15:26,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742119_1295 (size=12301) 2024-12-08T21:15:26,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:26,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:26,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:26,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T21:15:27,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:27,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:27,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:27,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:27,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692587097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:27,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692587098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:27,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/5a13afb3f70f461eb98311040b045082 2024-12-08T21:15:27,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/4acf26948ea54ff4948a9916a05f595b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4acf26948ea54ff4948a9916a05f595b 2024-12-08T21:15:27,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4acf26948ea54ff4948a9916a05f595b, entries=200, sequenceid=525, filesize=14.4 K 2024-12-08T21:15:27,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/ee9bd0af6ada4db69c3c1b5aead7502f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ee9bd0af6ada4db69c3c1b5aead7502f 2024-12-08T21:15:27,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ee9bd0af6ada4db69c3c1b5aead7502f, entries=150, sequenceid=525, filesize=12.0 K 2024-12-08T21:15:27,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/5a13afb3f70f461eb98311040b045082 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/5a13afb3f70f461eb98311040b045082 2024-12-08T21:15:27,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:27,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/5a13afb3f70f461eb98311040b045082, entries=150, sequenceid=525, filesize=12.0 K 2024-12-08T21:15:27,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~257.81 KB/264000, currentSize=87.22 KB/89310 for b7d9f8874d392b78c0ba8e541e909290 in 1254ms, sequenceid=525, compaction requested=true 2024-12-08T21:15:27,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:27,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:27,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:27,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:27,174 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:27,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:27,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:27,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:27,174 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:27,175 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,175 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,175 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/357fac638665462dba64f353280e704e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/8abc48c430b54ae4b7a2337f9fb87ed0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4acf26948ea54ff4948a9916a05f595b] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=39.1 K 2024-12-08T21:15:27,175 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/cdd8c10fb45349afba64902e02508245, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/257e64a765f54043a88d6b5f610aeadd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ee9bd0af6ada4db69c3c1b5aead7502f] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=36.7 K 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 357fac638665462dba64f353280e704e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733692522987 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd8c10fb45349afba64902e02508245, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733692522987 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 257e64a765f54043a88d6b5f610aeadd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733692525260 2024-12-08T21:15:27,175 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8abc48c430b54ae4b7a2337f9fb87ed0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733692525260 2024-12-08T21:15:27,176 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ee9bd0af6ada4db69c3c1b5aead7502f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733692525912 2024-12-08T21:15:27,176 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4acf26948ea54ff4948a9916a05f595b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733692525910 2024-12-08T21:15:27,184 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:27,184 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3d6b6cadcdcf47978b39a4f157a3dd8d is 50, key is test_row_0/A:col10/1733692525918/Put/seqid=0 2024-12-08T21:15:27,185 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:27,185 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/060ab5121c544a1fac1977997b4ad7b4 is 50, key is test_row_0/B:col10/1733692525918/Put/seqid=0 2024-12-08T21:15:27,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742120_1296 (size=13119) 2024-12-08T21:15:27,193 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3d6b6cadcdcf47978b39a4f157a3dd8d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3d6b6cadcdcf47978b39a4f157a3dd8d 2024-12-08T21:15:27,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742121_1297 (size=13119) 2024-12-08T21:15:27,198 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 3d6b6cadcdcf47978b39a4f157a3dd8d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:27,198 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:27,198 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=13, startTime=1733692527174; duration=0sec 2024-12-08T21:15:27,199 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:27,199 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:27,199 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:27,199 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:27,199 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:27,199 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,200 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/2faf7e2a801d471091ef5acd9f9027b2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2f1275027c463a8d4e67b660aaf92e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/5a13afb3f70f461eb98311040b045082] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=36.7 K 2024-12-08T21:15:27,200 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2faf7e2a801d471091ef5acd9f9027b2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1733692522987 2024-12-08T21:15:27,200 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2f1275027c463a8d4e67b660aaf92e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1733692525260 2024-12-08T21:15:27,200 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a13afb3f70f461eb98311040b045082, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733692525912 2024-12-08T21:15:27,207 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:27,208 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/d0b5ce3434574d8f9a5aaa28a3c00d7e is 50, key is test_row_0/C:col10/1733692525918/Put/seqid=0 2024-12-08T21:15:27,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742122_1298 (size=13119) 2024-12-08T21:15:27,220 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/d0b5ce3434574d8f9a5aaa28a3c00d7e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0b5ce3434574d8f9a5aaa28a3c00d7e 2024-12-08T21:15:27,225 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into d0b5ce3434574d8f9a5aaa28a3c00d7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:27,225 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:27,225 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=13, startTime=1733692527174; duration=0sec 2024-12-08T21:15:27,225 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:27,225 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:27,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-08T21:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:27,324 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T21:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:27,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:27,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/00c21b78963948cb92240f0b9f8c3b2c is 50, key is test_row_0/A:col10/1733692525940/Put/seqid=0 2024-12-08T21:15:27,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742123_1299 (size=12301) 2024-12-08T21:15:27,599 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/060ab5121c544a1fac1977997b4ad7b4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/060ab5121c544a1fac1977997b4ad7b4 2024-12-08T21:15:27,603 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into 060ab5121c544a1fac1977997b4ad7b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:27,603 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:27,604 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=13, startTime=1733692527174; duration=0sec 2024-12-08T21:15:27,604 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:27,604 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:27,733 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/00c21b78963948cb92240f0b9f8c3b2c 2024-12-08T21:15:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/98e838a1704c42a9b1be50ce189c9198 is 50, key is test_row_0/B:col10/1733692525940/Put/seqid=0 2024-12-08T21:15:27,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742124_1300 (size=12301) 2024-12-08T21:15:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T21:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:28,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692588127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692588129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,151 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/98e838a1704c42a9b1be50ce189c9198 2024-12-08T21:15:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/3e729b05c4fa4ba5ab75cd69546b2b79 is 50, key is test_row_0/C:col10/1733692525940/Put/seqid=0 2024-12-08T21:15:28,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742125_1301 (size=12301) 2024-12-08T21:15:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692588238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692588238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692588448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692588448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,567 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/3e729b05c4fa4ba5ab75cd69546b2b79 2024-12-08T21:15:28,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/00c21b78963948cb92240f0b9f8c3b2c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/00c21b78963948cb92240f0b9f8c3b2c 2024-12-08T21:15:28,575 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/00c21b78963948cb92240f0b9f8c3b2c, entries=150, sequenceid=543, filesize=12.0 K 2024-12-08T21:15:28,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/98e838a1704c42a9b1be50ce189c9198 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/98e838a1704c42a9b1be50ce189c9198 2024-12-08T21:15:28,579 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/98e838a1704c42a9b1be50ce189c9198, entries=150, sequenceid=543, filesize=12.0 K 2024-12-08T21:15:28,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/3e729b05c4fa4ba5ab75cd69546b2b79 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3e729b05c4fa4ba5ab75cd69546b2b79 2024-12-08T21:15:28,583 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3e729b05c4fa4ba5ab75cd69546b2b79, entries=150, sequenceid=543, filesize=12.0 K 2024-12-08T21:15:28,584 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7d9f8874d392b78c0ba8e541e909290 in 1260ms, sequenceid=543, compaction requested=false 2024-12-08T21:15:28,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:28,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:28,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-08T21:15:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-08T21:15:28,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-08T21:15:28,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7960 sec 2024-12-08T21:15:28,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.8020 sec 2024-12-08T21:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:28,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T21:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:28,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:28,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692588782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692588782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3c86850007af4395ad6d4c360553d4b0 is 50, key is test_row_0/A:col10/1733692528128/Put/seqid=0 2024-12-08T21:15:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742126_1302 (size=17181) 2024-12-08T21:15:28,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3c86850007af4395ad6d4c360553d4b0 2024-12-08T21:15:28,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/a240f009db70484c8e5b79a7a290d0ef is 50, key is test_row_0/B:col10/1733692528128/Put/seqid=0 2024-12-08T21:15:28,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742127_1303 (size=12301) 2024-12-08T21:15:28,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692588888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692588888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:28,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692589098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692589098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/a240f009db70484c8e5b79a7a290d0ef 2024-12-08T21:15:29,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/72c1e34b19b84e3ab89ad646ea445f16 is 50, key is test_row_0/C:col10/1733692528128/Put/seqid=0 2024-12-08T21:15:29,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742128_1304 (size=12301) 2024-12-08T21:15:29,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48198 deadline: 1733692589288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,291 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48272 deadline: 1733692589307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,310 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48208 deadline: 1733692589328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,330 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in b7d9f8874d392b78c0ba8e541e909290 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692589408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692589408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/72c1e34b19b84e3ab89ad646ea445f16 2024-12-08T21:15:29,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/3c86850007af4395ad6d4c360553d4b0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3c86850007af4395ad6d4c360553d4b0 2024-12-08T21:15:29,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3c86850007af4395ad6d4c360553d4b0, entries=250, sequenceid=562, filesize=16.8 K 2024-12-08T21:15:29,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/a240f009db70484c8e5b79a7a290d0ef as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a240f009db70484c8e5b79a7a290d0ef 2024-12-08T21:15:29,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a240f009db70484c8e5b79a7a290d0ef, entries=150, sequenceid=562, filesize=12.0 K 2024-12-08T21:15:29,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/72c1e34b19b84e3ab89ad646ea445f16 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/72c1e34b19b84e3ab89ad646ea445f16 2024-12-08T21:15:29,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/72c1e34b19b84e3ab89ad646ea445f16, entries=150, sequenceid=562, filesize=12.0 K 2024-12-08T21:15:29,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7d9f8874d392b78c0ba8e541e909290 in 896ms, sequenceid=562, compaction requested=true 2024-12-08T21:15:29,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:29,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:29,658 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:29,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:29,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:29,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:29,658 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:29,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:29,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:29,658 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:29,658 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:29,658 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:29,658 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:29,658 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:29,658 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:29,659 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3d6b6cadcdcf47978b39a4f157a3dd8d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/00c21b78963948cb92240f0b9f8c3b2c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3c86850007af4395ad6d4c360553d4b0] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=41.6 K 2024-12-08T21:15:29,659 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/060ab5121c544a1fac1977997b4ad7b4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/98e838a1704c42a9b1be50ce189c9198, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a240f009db70484c8e5b79a7a290d0ef] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=36.8 K 2024-12-08T21:15:29,660 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 060ab5121c544a1fac1977997b4ad7b4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733692525912 2024-12-08T21:15:29,662 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d6b6cadcdcf47978b39a4f157a3dd8d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733692525912 2024-12-08T21:15:29,663 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c21b78963948cb92240f0b9f8c3b2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1733692525940 2024-12-08T21:15:29,663 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 98e838a1704c42a9b1be50ce189c9198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1733692525940 2024-12-08T21:15:29,663 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a240f009db70484c8e5b79a7a290d0ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733692528128 2024-12-08T21:15:29,663 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c86850007af4395ad6d4c360553d4b0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733692528125 2024-12-08T21:15:29,671 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:29,671 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/226d5c948aad4635bda2241172453a08 is 50, key is test_row_0/A:col10/1733692528128/Put/seqid=0 2024-12-08T21:15:29,680 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:29,680 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/3f111de62e2f4ecb9d878d0c34eb8ccc is 50, key is test_row_0/B:col10/1733692528128/Put/seqid=0 2024-12-08T21:15:29,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742129_1305 (size=13221) 2024-12-08T21:15:29,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742130_1306 (size=13221) 2024-12-08T21:15:29,699 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/3f111de62e2f4ecb9d878d0c34eb8ccc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/3f111de62e2f4ecb9d878d0c34eb8ccc 2024-12-08T21:15:29,706 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into 3f111de62e2f4ecb9d878d0c34eb8ccc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:29,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:29,706 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=13, startTime=1733692529658; duration=0sec 2024-12-08T21:15:29,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:29,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:29,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:29,708 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:29,708 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:29,708 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:29,708 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0b5ce3434574d8f9a5aaa28a3c00d7e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3e729b05c4fa4ba5ab75cd69546b2b79, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/72c1e34b19b84e3ab89ad646ea445f16] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=36.8 K 2024-12-08T21:15:29,709 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting d0b5ce3434574d8f9a5aaa28a3c00d7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733692525912 2024-12-08T21:15:29,709 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e729b05c4fa4ba5ab75cd69546b2b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1733692525940 2024-12-08T21:15:29,710 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c1e34b19b84e3ab89ad646ea445f16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733692528128 2024-12-08T21:15:29,721 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:29,722 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/ba7fccb901064c4eb6e3709e302e98c7 is 50, key is test_row_0/C:col10/1733692528128/Put/seqid=0 2024-12-08T21:15:29,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742131_1307 (size=13221) 2024-12-08T21:15:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:29,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T21:15:29,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:29,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:29,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:29,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:29,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:29,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:29,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/5ef289adef724af4b6a2909167837db0 is 50, key is test_row_0/A:col10/1733692528773/Put/seqid=0 2024-12-08T21:15:29,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742132_1308 (size=17181) 2024-12-08T21:15:29,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/5ef289adef724af4b6a2909167837db0 2024-12-08T21:15:29,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0bc7a8ee9b174d66ae6c404ce667bdae is 50, key is test_row_0/B:col10/1733692528773/Put/seqid=0 2024-12-08T21:15:29,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742133_1309 (size=12301) 2024-12-08T21:15:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-08T21:15:29,947 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:29,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-08T21:15:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T21:15:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692589946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,949 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692589947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:29,950 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T21:15:30,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692590058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692590058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:30,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:30,092 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/226d5c948aad4635bda2241172453a08 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/226d5c948aad4635bda2241172453a08 2024-12-08T21:15:30,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T21:15:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,103 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 226d5c948aad4635bda2241172453a08(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:30,103 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:30,103 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=13, startTime=1733692529657; duration=0sec 2024-12-08T21:15:30,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,103 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:30,103 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:30,157 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/ba7fccb901064c4eb6e3709e302e98c7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/ba7fccb901064c4eb6e3709e302e98c7 2024-12-08T21:15:30,163 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into ba7fccb901064c4eb6e3709e302e98c7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:30,163 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:30,163 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=13, startTime=1733692529658; duration=0sec 2024-12-08T21:15:30,163 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:30,163 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:30,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T21:15:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T21:15:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692590268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:30,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692590268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:30,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:30,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0bc7a8ee9b174d66ae6c404ce667bdae 2024-12-08T21:15:30,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/4a8763184a0d4a438587d8f8ca998517 is 50, key is test_row_0/C:col10/1733692528773/Put/seqid=0 2024-12-08T21:15:30,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742134_1310 (size=12301) 2024-12-08T21:15:30,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T21:15:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T21:15:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T21:15:30,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:30,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692590578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692590578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:30,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:30,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T21:15:30,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:30,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:30,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/4a8763184a0d4a438587d8f8ca998517 2024-12-08T21:15:30,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/5ef289adef724af4b6a2909167837db0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/5ef289adef724af4b6a2909167837db0 2024-12-08T21:15:30,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/5ef289adef724af4b6a2909167837db0, entries=250, sequenceid=579, filesize=16.8 K 2024-12-08T21:15:30,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0bc7a8ee9b174d66ae6c404ce667bdae as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0bc7a8ee9b174d66ae6c404ce667bdae 2024-12-08T21:15:30,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0bc7a8ee9b174d66ae6c404ce667bdae, entries=150, sequenceid=579, filesize=12.0 K 2024-12-08T21:15:30,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/4a8763184a0d4a438587d8f8ca998517 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/4a8763184a0d4a438587d8f8ca998517 2024-12-08T21:15:30,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/4a8763184a0d4a438587d8f8ca998517, entries=150, sequenceid=579, filesize=12.0 K 2024-12-08T21:15:30,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7d9f8874d392b78c0ba8e541e909290 in 851ms, sequenceid=579, compaction requested=false 2024-12-08T21:15:30,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:30,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-08T21:15:30,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:30,868 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T21:15:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:30,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/9c9385a24614422ea8a9854a8716951a is 50, key is test_row_0/A:col10/1733692529942/Put/seqid=0 2024-12-08T21:15:30,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742135_1311 (size=12301) 2024-12-08T21:15:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T21:15:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:31,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:31,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692591112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692591115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692591227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692591228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,285 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/9c9385a24614422ea8a9854a8716951a 2024-12-08T21:15:31,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0574fa6bfd1044c7b801e4a969bd5d9d is 50, key is test_row_0/B:col10/1733692529942/Put/seqid=0 2024-12-08T21:15:31,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742136_1312 (size=12301) 2024-12-08T21:15:31,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0574fa6bfd1044c7b801e4a969bd5d9d 2024-12-08T21:15:31,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7cb0ba733c764c5696580554eef2c4aa is 50, key is test_row_0/C:col10/1733692529942/Put/seqid=0 2024-12-08T21:15:31,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742137_1313 (size=12301) 2024-12-08T21:15:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692591438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692591438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,718 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7cb0ba733c764c5696580554eef2c4aa 2024-12-08T21:15:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/9c9385a24614422ea8a9854a8716951a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/9c9385a24614422ea8a9854a8716951a 2024-12-08T21:15:31,734 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/9c9385a24614422ea8a9854a8716951a, entries=150, sequenceid=597, filesize=12.0 K 2024-12-08T21:15:31,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/0574fa6bfd1044c7b801e4a969bd5d9d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0574fa6bfd1044c7b801e4a969bd5d9d 2024-12-08T21:15:31,739 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0574fa6bfd1044c7b801e4a969bd5d9d, entries=150, sequenceid=597, filesize=12.0 K 2024-12-08T21:15:31,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/7cb0ba733c764c5696580554eef2c4aa as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7cb0ba733c764c5696580554eef2c4aa 2024-12-08T21:15:31,750 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7cb0ba733c764c5696580554eef2c4aa, entries=150, sequenceid=597, filesize=12.0 K 2024-12-08T21:15:31,751 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for b7d9f8874d392b78c0ba8e541e909290 in 883ms, sequenceid=597, compaction requested=true 2024-12-08T21:15:31,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:31,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:31,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-08T21:15:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-08T21:15:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:31,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T21:15:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:31,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-08T21:15:31,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8030 sec 2024-12-08T21:15:31,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c4d5d005f6264c44b7fd6b5d93b2552c is 50, key is test_row_0/A:col10/1733692531110/Put/seqid=0 2024-12-08T21:15:31,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.8120 sec 2024-12-08T21:15:31,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742138_1314 (size=14741) 2024-12-08T21:15:31,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c4d5d005f6264c44b7fd6b5d93b2552c 2024-12-08T21:15:31,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 331 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692591800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692591800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d76d7be6544947928a2b465725d670d7 is 50, key is test_row_0/B:col10/1733692531110/Put/seqid=0 2024-12-08T21:15:31,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742139_1315 (size=12301) 2024-12-08T21:15:31,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d76d7be6544947928a2b465725d670d7 2024-12-08T21:15:31,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/b8ec312f4b6c4183a20661efc6075105 is 50, key is test_row_0/C:col10/1733692531110/Put/seqid=0 2024-12-08T21:15:31,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742140_1316 (size=12301) 2024-12-08T21:15:31,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/b8ec312f4b6c4183a20661efc6075105 2024-12-08T21:15:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c4d5d005f6264c44b7fd6b5d93b2552c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c4d5d005f6264c44b7fd6b5d93b2552c 2024-12-08T21:15:31,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c4d5d005f6264c44b7fd6b5d93b2552c, entries=200, sequenceid=615, filesize=14.4 K 2024-12-08T21:15:31,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692591909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692591920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:31,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/d76d7be6544947928a2b465725d670d7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d76d7be6544947928a2b465725d670d7 2024-12-08T21:15:31,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d76d7be6544947928a2b465725d670d7, entries=150, sequenceid=615, filesize=12.0 K 2024-12-08T21:15:31,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/b8ec312f4b6c4183a20661efc6075105 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/b8ec312f4b6c4183a20661efc6075105 2024-12-08T21:15:31,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/b8ec312f4b6c4183a20661efc6075105, entries=150, sequenceid=615, filesize=12.0 K 2024-12-08T21:15:31,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for b7d9f8874d392b78c0ba8e541e909290 in 187ms, sequenceid=615, compaction requested=true 2024-12-08T21:15:31,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:31,942 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:15:31,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:31,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:31,943 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:15:31,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:31,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:31,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b7d9f8874d392b78c0ba8e541e909290:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:31,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:31,944 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:15:31,944 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:15:31,944 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/B is initiating minor compaction (all files) 2024-12-08T21:15:31,944 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/A is initiating minor compaction (all files) 2024-12-08T21:15:31,944 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/A in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:31,944 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/B in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:31,944 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/226d5c948aad4635bda2241172453a08, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/5ef289adef724af4b6a2909167837db0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/9c9385a24614422ea8a9854a8716951a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c4d5d005f6264c44b7fd6b5d93b2552c] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=56.1 K 2024-12-08T21:15:31,944 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/3f111de62e2f4ecb9d878d0c34eb8ccc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0bc7a8ee9b174d66ae6c404ce667bdae, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0574fa6bfd1044c7b801e4a969bd5d9d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d76d7be6544947928a2b465725d670d7] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=48.9 K 2024-12-08T21:15:31,944 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f111de62e2f4ecb9d878d0c34eb8ccc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733692528128 2024-12-08T21:15:31,944 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226d5c948aad4635bda2241172453a08, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733692528128 2024-12-08T21:15:31,945 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc7a8ee9b174d66ae6c404ce667bdae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733692528773 2024-12-08T21:15:31,945 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef289adef724af4b6a2909167837db0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733692528773 2024-12-08T21:15:31,945 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0574fa6bfd1044c7b801e4a969bd5d9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733692529937 2024-12-08T21:15:31,945 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c9385a24614422ea8a9854a8716951a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733692529937 2024-12-08T21:15:31,945 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4d5d005f6264c44b7fd6b5d93b2552c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733692531105 2024-12-08T21:15:31,945 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting d76d7be6544947928a2b465725d670d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733692531110 2024-12-08T21:15:31,968 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#B#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:31,969 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/2d7b8997ab5b4ac8b9fd206b97697176 is 50, key is test_row_0/B:col10/1733692531110/Put/seqid=0 2024-12-08T21:15:31,972 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#A#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:31,973 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/89de3b2d855e49ebaf35ca4201d1c35d is 50, key is test_row_0/A:col10/1733692531110/Put/seqid=0 2024-12-08T21:15:32,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742141_1317 (size=13357) 2024-12-08T21:15:32,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742142_1318 (size=13357) 2024-12-08T21:15:32,025 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/2d7b8997ab5b4ac8b9fd206b97697176 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d7b8997ab5b4ac8b9fd206b97697176 2024-12-08T21:15:32,031 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/89de3b2d855e49ebaf35ca4201d1c35d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/89de3b2d855e49ebaf35ca4201d1c35d 2024-12-08T21:15:32,035 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/B of b7d9f8874d392b78c0ba8e541e909290 into 2d7b8997ab5b4ac8b9fd206b97697176(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:32,035 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:32,035 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/B, priority=12, startTime=1733692531942; duration=0sec 2024-12-08T21:15:32,035 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:32,035 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:B 2024-12-08T21:15:32,035 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:15:32,038 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/A of b7d9f8874d392b78c0ba8e541e909290 into 89de3b2d855e49ebaf35ca4201d1c35d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:32,038 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:32,038 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/A, priority=12, startTime=1733692531942; duration=0sec 2024-12-08T21:15:32,038 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:32,038 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:A 2024-12-08T21:15:32,038 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:15:32,038 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): b7d9f8874d392b78c0ba8e541e909290/C is initiating minor compaction (all files) 2024-12-08T21:15:32,038 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b7d9f8874d392b78c0ba8e541e909290/C in TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,039 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/ba7fccb901064c4eb6e3709e302e98c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/4a8763184a0d4a438587d8f8ca998517, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7cb0ba733c764c5696580554eef2c4aa, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/b8ec312f4b6c4183a20661efc6075105] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp, totalSize=48.9 K 2024-12-08T21:15:32,039 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7fccb901064c4eb6e3709e302e98c7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1733692528128 2024-12-08T21:15:32,039 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a8763184a0d4a438587d8f8ca998517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733692528773 2024-12-08T21:15:32,040 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cb0ba733c764c5696580554eef2c4aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1733692529937 2024-12-08T21:15:32,040 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ec312f4b6c4183a20661efc6075105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=615, earliestPutTs=1733692531110 2024-12-08T21:15:32,051 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b7d9f8874d392b78c0ba8e541e909290#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:32,052 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/59f0c7b5e8cc43a9afe7c0a3b53649d7 is 50, key is test_row_0/C:col10/1733692531110/Put/seqid=0 2024-12-08T21:15:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742143_1319 (size=13357) 2024-12-08T21:15:32,083 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/59f0c7b5e8cc43a9afe7c0a3b53649d7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/59f0c7b5e8cc43a9afe7c0a3b53649d7 2024-12-08T21:15:32,091 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b7d9f8874d392b78c0ba8e541e909290/C of b7d9f8874d392b78c0ba8e541e909290 into 59f0c7b5e8cc43a9afe7c0a3b53649d7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:32,091 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:32,091 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., storeName=b7d9f8874d392b78c0ba8e541e909290/C, priority=12, startTime=1733692531943; duration=0sec 2024-12-08T21:15:32,092 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:32,092 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b7d9f8874d392b78c0ba8e541e909290:C 2024-12-08T21:15:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-08T21:15:32,098 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:32,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-08T21:15:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T21:15:32,102 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:32,103 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:32,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:32,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T21:15:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:32,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/df2cb76bf1294df9bd957b069dc393c6 is 50, key is test_row_0/A:col10/1733692531774/Put/seqid=0 2024-12-08T21:15:32,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742144_1320 (size=12301) 2024-12-08T21:15:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692592168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692592170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T21:15:32,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T21:15:32,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:32,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 343 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692592278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692592278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T21:15:32,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:32,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T21:15:32,442 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:15:32,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692592488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692592489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/df2cb76bf1294df9bd957b069dc393c6 2024-12-08T21:15:32,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/5e6e3b14049942d9bddf36e95e536601 is 50, key is test_row_0/B:col10/1733692531774/Put/seqid=0 2024-12-08T21:15:32,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T21:15:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:32,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742145_1321 (size=12301) 2024-12-08T21:15:32,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/5e6e3b14049942d9bddf36e95e536601 2024-12-08T21:15:32,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/8b046c2bc29d45cfa2cdfa7cb938eac3 is 50, key is test_row_0/C:col10/1733692531774/Put/seqid=0 2024-12-08T21:15:32,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742146_1322 (size=12301) 2024-12-08T21:15:32,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T21:15:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T21:15:32,764 INFO [Thread-1065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1065. 2024-12-08T21:15:32,764 DEBUG [Thread-1065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:32,764 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,764 DEBUG [Thread-1065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:32,766 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-12-08T21:15:32,766 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:32,766 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,766 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:32,767 INFO [Thread-1067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1067. 2024-12-08T21:15:32,767 DEBUG [Thread-1067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:32,767 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,767 DEBUG [Thread-1067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,768 INFO [Thread-1069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1069. 2024-12-08T21:15:32,768 DEBUG [Thread-1069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:32,768 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,768 DEBUG [Thread-1069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:32,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:32,769 INFO [Thread-1063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1063. 2024-12-08T21:15:32,769 DEBUG [Thread-1063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:32,769 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,769 DEBUG [Thread-1063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:32,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:32,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48240 deadline: 1733692592798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48226 deadline: 1733692592799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:15:32,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T21:15:32,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:32,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:33,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T21:15:33,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:33,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:33,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:33,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:33,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=633 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/8b046c2bc29d45cfa2cdfa7cb938eac3 2024-12-08T21:15:33,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/df2cb76bf1294df9bd957b069dc393c6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/df2cb76bf1294df9bd957b069dc393c6 2024-12-08T21:15:33,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/df2cb76bf1294df9bd957b069dc393c6, entries=150, sequenceid=633, filesize=12.0 K 2024-12-08T21:15:33,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/5e6e3b14049942d9bddf36e95e536601 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5e6e3b14049942d9bddf36e95e536601 2024-12-08T21:15:33,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5e6e3b14049942d9bddf36e95e536601, entries=150, sequenceid=633, filesize=12.0 K 2024-12-08T21:15:33,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/8b046c2bc29d45cfa2cdfa7cb938eac3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/8b046c2bc29d45cfa2cdfa7cb938eac3 2024-12-08T21:15:33,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/8b046c2bc29d45cfa2cdfa7cb938eac3, entries=150, sequenceid=633, filesize=12.0 K 2024-12-08T21:15:33,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for b7d9f8874d392b78c0ba8e541e909290 in 951ms, sequenceid=633, compaction requested=false 2024-12-08T21:15:33,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:33,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-08T21:15:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:33,181 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T21:15:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:33,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:33,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/db3ca3c557924e58833e8bfbfbd2d1df is 50, key is test_row_0/A:col10/1733692532166/Put/seqid=0 2024-12-08T21:15:33,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742147_1323 (size=12301) 2024-12-08T21:15:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T21:15:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:33,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. as already flushing 2024-12-08T21:15:33,310 INFO [Thread-1054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1054. 2024-12-08T21:15:33,310 INFO [Thread-1056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1056. 2024-12-08T21:15:33,310 DEBUG [Thread-1056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:33,310 DEBUG [Thread-1054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:33,310 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:33,310 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:33,310 DEBUG [Thread-1056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:33,310 DEBUG [Thread-1054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:33,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:33,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:33,591 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=651 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/db3ca3c557924e58833e8bfbfbd2d1df 2024-12-08T21:15:33,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/c3bfec22d96b4b2d9c7613f1ed528666 is 50, key is test_row_0/B:col10/1733692532166/Put/seqid=0 2024-12-08T21:15:33,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742148_1324 (size=12301) 2024-12-08T21:15:34,008 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=651 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/c3bfec22d96b4b2d9c7613f1ed528666 2024-12-08T21:15:34,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/903238706dff4d30a5a82233c6a04353 is 50, key is test_row_0/C:col10/1733692532166/Put/seqid=0 2024-12-08T21:15:34,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742149_1325 (size=12301) 2024-12-08T21:15:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T21:15:34,425 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=651 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/903238706dff4d30a5a82233c6a04353 2024-12-08T21:15:34,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/db3ca3c557924e58833e8bfbfbd2d1df as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/db3ca3c557924e58833e8bfbfbd2d1df 2024-12-08T21:15:34,438 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/db3ca3c557924e58833e8bfbfbd2d1df, entries=150, sequenceid=651, filesize=12.0 K 2024-12-08T21:15:34,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/c3bfec22d96b4b2d9c7613f1ed528666 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/c3bfec22d96b4b2d9c7613f1ed528666 2024-12-08T21:15:34,443 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/c3bfec22d96b4b2d9c7613f1ed528666, entries=150, sequenceid=651, filesize=12.0 K 2024-12-08T21:15:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/903238706dff4d30a5a82233c6a04353 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/903238706dff4d30a5a82233c6a04353 2024-12-08T21:15:34,449 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/903238706dff4d30a5a82233c6a04353, entries=150, sequenceid=651, filesize=12.0 K 2024-12-08T21:15:34,450 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for b7d9f8874d392b78c0ba8e541e909290 in 1269ms, sequenceid=651, compaction requested=true 2024-12-08T21:15:34,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for b7d9f8874d392b78c0ba8e541e909290: 2024-12-08T21:15:34,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:34,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-08T21:15:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-08T21:15:34,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-08T21:15:34,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3480 sec 2024-12-08T21:15:34,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3550 sec 2024-12-08T21:15:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-08T21:15:36,259 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:39,350 INFO [Thread-1058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1058. 2024-12-08T21:15:39,350 DEBUG [Thread-1058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:39,350 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,350 DEBUG [Thread-1058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:39,401 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-12-08T21:15:39,401 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:39,401 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,401 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:39,429 INFO [Thread-1052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1052. 2024-12-08T21:15:39,429 DEBUG [Thread-1052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:15:39,430 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,430 DEBUG [Thread-1052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 279 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 230 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4159 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4064 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4142 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4124 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4095 2024-12-08T21:15:39,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T21:15:39,430 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:15:39,430 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:15:39,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:39,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:15:39,431 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T21:15:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T21:15:39,435 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692539435"}]},"ts":"1733692539435"} 2024-12-08T21:15:39,437 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T21:15:39,437 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T21:15:39,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:15:39,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, UNASSIGN}] 2024-12-08T21:15:39,442 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, UNASSIGN 2024-12-08T21:15:39,443 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b7d9f8874d392b78c0ba8e541e909290, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:39,444 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, UNASSIGN because future has completed 2024-12-08T21:15:39,444 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:15:39,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:15:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T21:15:39,597 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:39,597 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:15:39,597 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing b7d9f8874d392b78c0ba8e541e909290, disabling compactions & flushes 2024-12-08T21:15:39,597 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:39,598 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:39,598 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. after waiting 0 ms 2024-12-08T21:15:39,598 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:39,598 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing b7d9f8874d392b78c0ba8e541e909290 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T21:15:39,598 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=A 2024-12-08T21:15:39,599 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:39,599 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=B 2024-12-08T21:15:39,599 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:39,599 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b7d9f8874d392b78c0ba8e541e909290, store=C 2024-12-08T21:15:39,599 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:39,606 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c8d2cf90ecb84e9598ef309de36a86cb is 50, key is test_row_0/A:col10/1733692539348/Put/seqid=0 2024-12-08T21:15:39,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742150_1326 (size=12301) 2024-12-08T21:15:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T21:15:40,013 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c8d2cf90ecb84e9598ef309de36a86cb 2024-12-08T21:15:40,020 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7e7be39a5ede4725a7c3ecae13ce70b1 is 50, key is test_row_0/B:col10/1733692539348/Put/seqid=0 2024-12-08T21:15:40,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742151_1327 (size=12301) 2024-12-08T21:15:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T21:15:40,426 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7e7be39a5ede4725a7c3ecae13ce70b1 2024-12-08T21:15:40,438 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/20ea4d309766424eab0d8224a80a51f0 is 50, key is test_row_0/C:col10/1733692539348/Put/seqid=0 2024-12-08T21:15:40,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742152_1328 (size=12301) 2024-12-08T21:15:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T21:15:40,843 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=659 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/20ea4d309766424eab0d8224a80a51f0 2024-12-08T21:15:40,847 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/A/c8d2cf90ecb84e9598ef309de36a86cb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c8d2cf90ecb84e9598ef309de36a86cb 2024-12-08T21:15:40,851 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c8d2cf90ecb84e9598ef309de36a86cb, entries=150, sequenceid=659, filesize=12.0 K 2024-12-08T21:15:40,852 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/B/7e7be39a5ede4725a7c3ecae13ce70b1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7e7be39a5ede4725a7c3ecae13ce70b1 2024-12-08T21:15:40,855 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7e7be39a5ede4725a7c3ecae13ce70b1, entries=150, sequenceid=659, filesize=12.0 K 2024-12-08T21:15:40,856 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/.tmp/C/20ea4d309766424eab0d8224a80a51f0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/20ea4d309766424eab0d8224a80a51f0 2024-12-08T21:15:40,860 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/20ea4d309766424eab0d8224a80a51f0, entries=150, sequenceid=659, filesize=12.0 K 2024-12-08T21:15:40,860 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b7d9f8874d392b78c0ba8e541e909290 in 1262ms, sequenceid=659, compaction requested=true 2024-12-08T21:15:40,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/47d00e4576724f978b89a60e55ca23e9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/cd991e8624e54e068950add7827501f8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c6b417d43b264648bc08c6d7910ae646, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/475222b469c04dd48d3dfc86807d33e8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/7c7211558d9041b584088ec55a340ca2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/64e3136f341a4f34a1794106411afd75, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3781eca2fc3e43ebb83c2e8854f42a28, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/87451cfe62ab433180b32654aa947a25, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/737ec47afc374157b3bf0d280cc7b3ce, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/66cf2aee4a1c40d981903a98ad5d9a14, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/70a570d16ce144b3834e08f8f947bbb9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/b82dec05c6d7412bb6b7b4789609b6ab, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/35b6a6bc20d440828cb1e243cd3090bb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/ed846df795e14be39df054b9da1de0e4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/353ee7baf18d46aaa56a27d863c9343b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/761ecf5b23ac4a918e60e4bde154de07, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0664e2b4c10c4c1aaad1c4f15ec534b4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4217160411ae4253a1581601cec1cc77, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/2c5a987a08a64508952979567faa19b5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/aa9babfe41774703b92dd9b202ef2e90, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/357fac638665462dba64f353280e704e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0ceff38d8b04497a9d3c9edc701f0e8e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/8abc48c430b54ae4b7a2337f9fb87ed0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4acf26948ea54ff4948a9916a05f595b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3d6b6cadcdcf47978b39a4f157a3dd8d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/00c21b78963948cb92240f0b9f8c3b2c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3c86850007af4395ad6d4c360553d4b0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/226d5c948aad4635bda2241172453a08, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/5ef289adef724af4b6a2909167837db0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/9c9385a24614422ea8a9854a8716951a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c4d5d005f6264c44b7fd6b5d93b2552c] to archive 2024-12-08T21:15:40,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:15:40,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/47d00e4576724f978b89a60e55ca23e9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/47d00e4576724f978b89a60e55ca23e9 2024-12-08T21:15:40,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/cd991e8624e54e068950add7827501f8 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/cd991e8624e54e068950add7827501f8 2024-12-08T21:15:40,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c6b417d43b264648bc08c6d7910ae646 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c6b417d43b264648bc08c6d7910ae646 2024-12-08T21:15:40,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/475222b469c04dd48d3dfc86807d33e8 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/475222b469c04dd48d3dfc86807d33e8 2024-12-08T21:15:40,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/7c7211558d9041b584088ec55a340ca2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/7c7211558d9041b584088ec55a340ca2 2024-12-08T21:15:40,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/64e3136f341a4f34a1794106411afd75 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/64e3136f341a4f34a1794106411afd75 2024-12-08T21:15:40,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3781eca2fc3e43ebb83c2e8854f42a28 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3781eca2fc3e43ebb83c2e8854f42a28 2024-12-08T21:15:40,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/87451cfe62ab433180b32654aa947a25 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/87451cfe62ab433180b32654aa947a25 2024-12-08T21:15:40,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/737ec47afc374157b3bf0d280cc7b3ce to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/737ec47afc374157b3bf0d280cc7b3ce 2024-12-08T21:15:40,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/66cf2aee4a1c40d981903a98ad5d9a14 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/66cf2aee4a1c40d981903a98ad5d9a14 2024-12-08T21:15:40,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/70a570d16ce144b3834e08f8f947bbb9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/70a570d16ce144b3834e08f8f947bbb9 2024-12-08T21:15:40,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/b82dec05c6d7412bb6b7b4789609b6ab to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/b82dec05c6d7412bb6b7b4789609b6ab 2024-12-08T21:15:40,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/35b6a6bc20d440828cb1e243cd3090bb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/35b6a6bc20d440828cb1e243cd3090bb 2024-12-08T21:15:40,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/ed846df795e14be39df054b9da1de0e4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/ed846df795e14be39df054b9da1de0e4 2024-12-08T21:15:40,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/353ee7baf18d46aaa56a27d863c9343b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/353ee7baf18d46aaa56a27d863c9343b 2024-12-08T21:15:40,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/761ecf5b23ac4a918e60e4bde154de07 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/761ecf5b23ac4a918e60e4bde154de07 2024-12-08T21:15:40,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0664e2b4c10c4c1aaad1c4f15ec534b4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0664e2b4c10c4c1aaad1c4f15ec534b4 2024-12-08T21:15:40,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4217160411ae4253a1581601cec1cc77 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4217160411ae4253a1581601cec1cc77 2024-12-08T21:15:40,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/2c5a987a08a64508952979567faa19b5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/2c5a987a08a64508952979567faa19b5 2024-12-08T21:15:40,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/aa9babfe41774703b92dd9b202ef2e90 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/aa9babfe41774703b92dd9b202ef2e90 2024-12-08T21:15:40,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/357fac638665462dba64f353280e704e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/357fac638665462dba64f353280e704e 2024-12-08T21:15:40,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0ceff38d8b04497a9d3c9edc701f0e8e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/0ceff38d8b04497a9d3c9edc701f0e8e 2024-12-08T21:15:40,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/8abc48c430b54ae4b7a2337f9fb87ed0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/8abc48c430b54ae4b7a2337f9fb87ed0 2024-12-08T21:15:40,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4acf26948ea54ff4948a9916a05f595b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/4acf26948ea54ff4948a9916a05f595b 2024-12-08T21:15:40,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3d6b6cadcdcf47978b39a4f157a3dd8d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3d6b6cadcdcf47978b39a4f157a3dd8d 2024-12-08T21:15:40,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/00c21b78963948cb92240f0b9f8c3b2c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/00c21b78963948cb92240f0b9f8c3b2c 2024-12-08T21:15:40,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3c86850007af4395ad6d4c360553d4b0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/3c86850007af4395ad6d4c360553d4b0 2024-12-08T21:15:40,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/226d5c948aad4635bda2241172453a08 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/226d5c948aad4635bda2241172453a08 2024-12-08T21:15:40,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/5ef289adef724af4b6a2909167837db0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/5ef289adef724af4b6a2909167837db0 2024-12-08T21:15:40,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/9c9385a24614422ea8a9854a8716951a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/9c9385a24614422ea8a9854a8716951a 2024-12-08T21:15:40,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c4d5d005f6264c44b7fd6b5d93b2552c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c4d5d005f6264c44b7fd6b5d93b2552c 2024-12-08T21:15:40,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ccbddf5acaa244f0b27f56343b64c741, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/b9169cd519a74089aade947246ca846f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/15f3ff0faff0416badabbd7f8c2ab8c6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/029eeef15ee943429ffae6fab4d1b4e9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/184d38cbbf7a4614bf8119be54f26640, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a5d6e0a1bbbe4577829f53fe7a00f74b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d245f4a20bd41fbbd7f2c17c354dda9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/343ab9d9485c4617af7be421b8ede725, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d6b62c57ed1b4acdb1d8397e7dec3097, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/f939440f332d4cc08042801aacba8b0d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d49f368a7e3c4822a88d9806a6dd5bfd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/4a983edd64ee4285b68cceaa2d6cee5b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5ad40511c7fa4ef681ac4ea6c02e39db, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d04c07257a254b548be87c7607d1a502, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bed89cf0b89447b2ab7faa06778dded9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7eaf840fe0994790ab24f8cdbf688976, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bfbd18f8f4234aedb5ccdbe4b11c9d80, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/93acd9396892432b988d9bc027c36261, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0589007af3fd4466a50081b21fb9d5b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/cdd8c10fb45349afba64902e02508245, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/9abf03055e7a430d9812effa35b50654, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/257e64a765f54043a88d6b5f610aeadd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/060ab5121c544a1fac1977997b4ad7b4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ee9bd0af6ada4db69c3c1b5aead7502f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/98e838a1704c42a9b1be50ce189c9198, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/3f111de62e2f4ecb9d878d0c34eb8ccc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a240f009db70484c8e5b79a7a290d0ef, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0bc7a8ee9b174d66ae6c404ce667bdae, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0574fa6bfd1044c7b801e4a969bd5d9d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d76d7be6544947928a2b465725d670d7] to archive 2024-12-08T21:15:40,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:15:40,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ccbddf5acaa244f0b27f56343b64c741 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ccbddf5acaa244f0b27f56343b64c741 2024-12-08T21:15:40,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/b9169cd519a74089aade947246ca846f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/b9169cd519a74089aade947246ca846f 2024-12-08T21:15:40,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/15f3ff0faff0416badabbd7f8c2ab8c6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/15f3ff0faff0416badabbd7f8c2ab8c6 2024-12-08T21:15:40,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/029eeef15ee943429ffae6fab4d1b4e9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/029eeef15ee943429ffae6fab4d1b4e9 2024-12-08T21:15:40,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/184d38cbbf7a4614bf8119be54f26640 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/184d38cbbf7a4614bf8119be54f26640 2024-12-08T21:15:40,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a5d6e0a1bbbe4577829f53fe7a00f74b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a5d6e0a1bbbe4577829f53fe7a00f74b 2024-12-08T21:15:40,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7a54ae96e8cf4ebcb7d14df5a7c3b22d 2024-12-08T21:15:40,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d245f4a20bd41fbbd7f2c17c354dda9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d245f4a20bd41fbbd7f2c17c354dda9 2024-12-08T21:15:40,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/343ab9d9485c4617af7be421b8ede725 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/343ab9d9485c4617af7be421b8ede725 2024-12-08T21:15:40,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d6b62c57ed1b4acdb1d8397e7dec3097 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d6b62c57ed1b4acdb1d8397e7dec3097 2024-12-08T21:15:40,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/f939440f332d4cc08042801aacba8b0d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/f939440f332d4cc08042801aacba8b0d 2024-12-08T21:15:40,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d49f368a7e3c4822a88d9806a6dd5bfd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d49f368a7e3c4822a88d9806a6dd5bfd 2024-12-08T21:15:40,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/4a983edd64ee4285b68cceaa2d6cee5b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/4a983edd64ee4285b68cceaa2d6cee5b 2024-12-08T21:15:40,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5ad40511c7fa4ef681ac4ea6c02e39db to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5ad40511c7fa4ef681ac4ea6c02e39db 2024-12-08T21:15:40,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d04c07257a254b548be87c7607d1a502 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d04c07257a254b548be87c7607d1a502 2024-12-08T21:15:40,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bed89cf0b89447b2ab7faa06778dded9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bed89cf0b89447b2ab7faa06778dded9 2024-12-08T21:15:40,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7eaf840fe0994790ab24f8cdbf688976 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7eaf840fe0994790ab24f8cdbf688976 2024-12-08T21:15:40,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bfbd18f8f4234aedb5ccdbe4b11c9d80 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/bfbd18f8f4234aedb5ccdbe4b11c9d80 2024-12-08T21:15:40,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/93acd9396892432b988d9bc027c36261 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/93acd9396892432b988d9bc027c36261 2024-12-08T21:15:40,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0589007af3fd4466a50081b21fb9d5b7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0589007af3fd4466a50081b21fb9d5b7 2024-12-08T21:15:40,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/cdd8c10fb45349afba64902e02508245 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/cdd8c10fb45349afba64902e02508245 2024-12-08T21:15:40,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/9abf03055e7a430d9812effa35b50654 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/9abf03055e7a430d9812effa35b50654 2024-12-08T21:15:40,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/257e64a765f54043a88d6b5f610aeadd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/257e64a765f54043a88d6b5f610aeadd 2024-12-08T21:15:40,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/060ab5121c544a1fac1977997b4ad7b4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/060ab5121c544a1fac1977997b4ad7b4 2024-12-08T21:15:40,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ee9bd0af6ada4db69c3c1b5aead7502f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/ee9bd0af6ada4db69c3c1b5aead7502f 2024-12-08T21:15:40,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/98e838a1704c42a9b1be50ce189c9198 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/98e838a1704c42a9b1be50ce189c9198 2024-12-08T21:15:40,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/3f111de62e2f4ecb9d878d0c34eb8ccc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/3f111de62e2f4ecb9d878d0c34eb8ccc 2024-12-08T21:15:40,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a240f009db70484c8e5b79a7a290d0ef to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/a240f009db70484c8e5b79a7a290d0ef 2024-12-08T21:15:40,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0bc7a8ee9b174d66ae6c404ce667bdae to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0bc7a8ee9b174d66ae6c404ce667bdae 2024-12-08T21:15:40,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0574fa6bfd1044c7b801e4a969bd5d9d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/0574fa6bfd1044c7b801e4a969bd5d9d 2024-12-08T21:15:40,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d76d7be6544947928a2b465725d670d7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/d76d7be6544947928a2b465725d670d7 2024-12-08T21:15:40,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/cd0f60d0f95e4cc5afccaa8dd1e27316, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/151aee0698414259bc31d0af3ae8e078, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2c17f30f9b4fc6ae1a5a87569f011a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/354ece3f707544eaa5f749749c5763b2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/e65098322e40417c8ec8d55df9d9759a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7d68953e0acb4966a8a53b91720f8900, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a813fbbeb4a84024a7a1ba034d8feb88, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7f7b12219e1c459d904ddaddd800b7c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d45878a59ccf40679c0bd3e45214c94c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/74921ab3c28f43d0bd98d203584423f1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/917f31595e7745f89e5ce4562ea25f9d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0d08b400d4c4698a7ec30e0cf2158eb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/19d6e9efa8634debbc177c35c8701fef, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a3af63ba7f2946248c6bfeec482807f6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/82f6c1bdf6cf494e86e6041933c5db87, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/891dca7c3d8d4965abe3193d18116e68, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/05779387652348d2be3bc5a2d1554488, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/02cb32043be043b5b498b9b66bf50071, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/79df9e842c4f4303b4f30e8bceb560df, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/f64bd176c1e145689e302380706ec66c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/2faf7e2a801d471091ef5acd9f9027b2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3d2eddd71cac46f8addfb3f90776e8aa, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2f1275027c463a8d4e67b660aaf92e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0b5ce3434574d8f9a5aaa28a3c00d7e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/5a13afb3f70f461eb98311040b045082, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3e729b05c4fa4ba5ab75cd69546b2b79, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/ba7fccb901064c4eb6e3709e302e98c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/72c1e34b19b84e3ab89ad646ea445f16, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/4a8763184a0d4a438587d8f8ca998517, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7cb0ba733c764c5696580554eef2c4aa, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/b8ec312f4b6c4183a20661efc6075105] to archive 2024-12-08T21:15:40,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:15:40,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/cd0f60d0f95e4cc5afccaa8dd1e27316 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/cd0f60d0f95e4cc5afccaa8dd1e27316 2024-12-08T21:15:40,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/151aee0698414259bc31d0af3ae8e078 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/151aee0698414259bc31d0af3ae8e078 2024-12-08T21:15:40,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2c17f30f9b4fc6ae1a5a87569f011a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2c17f30f9b4fc6ae1a5a87569f011a 2024-12-08T21:15:40,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/354ece3f707544eaa5f749749c5763b2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/354ece3f707544eaa5f749749c5763b2 2024-12-08T21:15:40,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/e65098322e40417c8ec8d55df9d9759a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/e65098322e40417c8ec8d55df9d9759a 2024-12-08T21:15:40,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7d68953e0acb4966a8a53b91720f8900 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7d68953e0acb4966a8a53b91720f8900 2024-12-08T21:15:40,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a813fbbeb4a84024a7a1ba034d8feb88 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a813fbbeb4a84024a7a1ba034d8feb88 2024-12-08T21:15:40,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7f7b12219e1c459d904ddaddd800b7c7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7f7b12219e1c459d904ddaddd800b7c7 2024-12-08T21:15:40,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d45878a59ccf40679c0bd3e45214c94c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d45878a59ccf40679c0bd3e45214c94c 2024-12-08T21:15:40,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/74921ab3c28f43d0bd98d203584423f1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/74921ab3c28f43d0bd98d203584423f1 2024-12-08T21:15:40,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/917f31595e7745f89e5ce4562ea25f9d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/917f31595e7745f89e5ce4562ea25f9d 2024-12-08T21:15:40,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0d08b400d4c4698a7ec30e0cf2158eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0d08b400d4c4698a7ec30e0cf2158eb 2024-12-08T21:15:40,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/19d6e9efa8634debbc177c35c8701fef to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/19d6e9efa8634debbc177c35c8701fef 2024-12-08T21:15:40,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a3af63ba7f2946248c6bfeec482807f6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/a3af63ba7f2946248c6bfeec482807f6 2024-12-08T21:15:40,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/82f6c1bdf6cf494e86e6041933c5db87 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/82f6c1bdf6cf494e86e6041933c5db87 2024-12-08T21:15:40,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/891dca7c3d8d4965abe3193d18116e68 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/891dca7c3d8d4965abe3193d18116e68 2024-12-08T21:15:40,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/05779387652348d2be3bc5a2d1554488 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/05779387652348d2be3bc5a2d1554488 2024-12-08T21:15:40,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/02cb32043be043b5b498b9b66bf50071 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/02cb32043be043b5b498b9b66bf50071 2024-12-08T21:15:40,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/79df9e842c4f4303b4f30e8bceb560df to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/79df9e842c4f4303b4f30e8bceb560df 2024-12-08T21:15:40,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/f64bd176c1e145689e302380706ec66c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/f64bd176c1e145689e302380706ec66c 2024-12-08T21:15:40,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/2faf7e2a801d471091ef5acd9f9027b2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/2faf7e2a801d471091ef5acd9f9027b2 2024-12-08T21:15:40,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3d2eddd71cac46f8addfb3f90776e8aa to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3d2eddd71cac46f8addfb3f90776e8aa 2024-12-08T21:15:40,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2f1275027c463a8d4e67b660aaf92e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7b2f1275027c463a8d4e67b660aaf92e 2024-12-08T21:15:40,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0b5ce3434574d8f9a5aaa28a3c00d7e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/d0b5ce3434574d8f9a5aaa28a3c00d7e 2024-12-08T21:15:40,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/5a13afb3f70f461eb98311040b045082 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/5a13afb3f70f461eb98311040b045082 2024-12-08T21:15:40,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3e729b05c4fa4ba5ab75cd69546b2b79 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/3e729b05c4fa4ba5ab75cd69546b2b79 2024-12-08T21:15:40,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/ba7fccb901064c4eb6e3709e302e98c7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/ba7fccb901064c4eb6e3709e302e98c7 2024-12-08T21:15:40,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/72c1e34b19b84e3ab89ad646ea445f16 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/72c1e34b19b84e3ab89ad646ea445f16 2024-12-08T21:15:40,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/4a8763184a0d4a438587d8f8ca998517 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/4a8763184a0d4a438587d8f8ca998517 2024-12-08T21:15:40,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7cb0ba733c764c5696580554eef2c4aa to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/7cb0ba733c764c5696580554eef2c4aa 2024-12-08T21:15:40,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/b8ec312f4b6c4183a20661efc6075105 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/b8ec312f4b6c4183a20661efc6075105 2024-12-08T21:15:40,948 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/recovered.edits/662.seqid, newMaxSeqId=662, maxSeqId=1 2024-12-08T21:15:40,948 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. 2024-12-08T21:15:40,948 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for b7d9f8874d392b78c0ba8e541e909290: Waiting for close lock at 1733692539597Running coprocessor pre-close hooks at 1733692539597Disabling compacts and flushes for region at 1733692539597Disabling writes for close at 1733692539598 (+1 ms)Obtaining lock to block concurrent updates at 1733692539598Preparing flush snapshotting stores in b7d9f8874d392b78c0ba8e541e909290 at 1733692539598Finished memstore snapshotting TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733692539599 (+1 ms)Flushing stores of TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290. at 1733692539600 (+1 ms)Flushing b7d9f8874d392b78c0ba8e541e909290/A: creating writer at 1733692539600Flushing b7d9f8874d392b78c0ba8e541e909290/A: appending metadata at 1733692539605 (+5 ms)Flushing b7d9f8874d392b78c0ba8e541e909290/A: closing flushed file at 1733692539605Flushing b7d9f8874d392b78c0ba8e541e909290/B: creating writer at 1733692540017 (+412 ms)Flushing b7d9f8874d392b78c0ba8e541e909290/B: appending metadata at 1733692540020 (+3 ms)Flushing b7d9f8874d392b78c0ba8e541e909290/B: closing flushed file at 1733692540020Flushing b7d9f8874d392b78c0ba8e541e909290/C: creating writer at 1733692540434 (+414 ms)Flushing b7d9f8874d392b78c0ba8e541e909290/C: appending metadata at 1733692540438 (+4 ms)Flushing b7d9f8874d392b78c0ba8e541e909290/C: closing flushed file at 1733692540438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7da2f6ca: reopening flushed file at 1733692540846 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c27afa5: reopening flushed file at 1733692540851 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ab855c5: reopening flushed file at 1733692540856 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for b7d9f8874d392b78c0ba8e541e909290 in 1262ms, sequenceid=659, compaction requested=true at 1733692540860 (+4 ms)Writing region close event to WAL at 1733692540945 (+85 ms)Running coprocessor post-close hooks at 1733692540948 (+3 ms)Closed at 1733692540948 2024-12-08T21:15:40,950 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:40,950 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b7d9f8874d392b78c0ba8e541e909290, regionState=CLOSED 2024-12-08T21:15:40,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:15:40,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-08T21:15:40,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure b7d9f8874d392b78c0ba8e541e909290, server=e653e208f502,38457,1733692444075 in 1.5080 sec 2024-12-08T21:15:40,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-08T21:15:40,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b7d9f8874d392b78c0ba8e541e909290, UNASSIGN in 1.5130 sec 2024-12-08T21:15:40,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-08T21:15:40,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5170 sec 2024-12-08T21:15:40,957 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692540957"}]},"ts":"1733692540957"} 2024-12-08T21:15:40,958 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T21:15:40,958 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T21:15:40,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5280 sec 2024-12-08T21:15:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-08T21:15:41,579 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:41,580 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T21:15:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:41,583 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T21:15:41,585 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:41,588 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:41,590 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/recovered.edits] 2024-12-08T21:15:41,594 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/89de3b2d855e49ebaf35ca4201d1c35d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/89de3b2d855e49ebaf35ca4201d1c35d 2024-12-08T21:15:41,595 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c8d2cf90ecb84e9598ef309de36a86cb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/c8d2cf90ecb84e9598ef309de36a86cb 2024-12-08T21:15:41,596 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/db3ca3c557924e58833e8bfbfbd2d1df to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/db3ca3c557924e58833e8bfbfbd2d1df 2024-12-08T21:15:41,598 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/df2cb76bf1294df9bd957b069dc393c6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/A/df2cb76bf1294df9bd957b069dc393c6 2024-12-08T21:15:41,602 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d7b8997ab5b4ac8b9fd206b97697176 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/2d7b8997ab5b4ac8b9fd206b97697176 2024-12-08T21:15:41,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5e6e3b14049942d9bddf36e95e536601 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/5e6e3b14049942d9bddf36e95e536601 2024-12-08T21:15:41,606 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7e7be39a5ede4725a7c3ecae13ce70b1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/7e7be39a5ede4725a7c3ecae13ce70b1 2024-12-08T21:15:41,608 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/c3bfec22d96b4b2d9c7613f1ed528666 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/B/c3bfec22d96b4b2d9c7613f1ed528666 2024-12-08T21:15:41,611 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/20ea4d309766424eab0d8224a80a51f0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/20ea4d309766424eab0d8224a80a51f0 2024-12-08T21:15:41,613 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/59f0c7b5e8cc43a9afe7c0a3b53649d7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/59f0c7b5e8cc43a9afe7c0a3b53649d7 2024-12-08T21:15:41,615 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/8b046c2bc29d45cfa2cdfa7cb938eac3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/8b046c2bc29d45cfa2cdfa7cb938eac3 2024-12-08T21:15:41,616 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/903238706dff4d30a5a82233c6a04353 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/C/903238706dff4d30a5a82233c6a04353 2024-12-08T21:15:41,618 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/recovered.edits/662.seqid to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290/recovered.edits/662.seqid 2024-12-08T21:15:41,619 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/b7d9f8874d392b78c0ba8e541e909290 2024-12-08T21:15:41,619 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T21:15:41,621 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:41,623 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T21:15:41,625 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T21:15:41,626 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:41,626 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T21:15:41,626 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733692541626"}]},"ts":"9223372036854775807"} 2024-12-08T21:15:41,627 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T21:15:41,627 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b7d9f8874d392b78c0ba8e541e909290, NAME => 'TestAcidGuarantees,,1733692510563.b7d9f8874d392b78c0ba8e541e909290.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T21:15:41,628 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T21:15:41,628 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733692541628"}]},"ts":"9223372036854775807"} 2024-12-08T21:15:41,629 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T21:15:41,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:41,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-12-08T21:15:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-08T21:15:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T21:15:41,691 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:41,698 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 373) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7860 (was 8636) 2024-12-08T21:15:41,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=7860 2024-12-08T21:15:41,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:15:41,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:15:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:41,710 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:15:41,710 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:41,711 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-12-08T21:15:41,711 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:15:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T21:15:41,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742153_1329 (size=1061) 2024-12-08T21:15:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T21:15:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T21:15:42,121 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:15:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742154_1330 (size=53) 2024-12-08T21:15:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T21:15:42,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:15:42,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4fdcd6355ad16589e9afb4e1e4d173eb, disabling compactions & flushes 2024-12-08T21:15:42,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. after waiting 0 ms 2024-12-08T21:15:42,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4fdcd6355ad16589e9afb4e1e4d173eb: Waiting for close lock at 1733692542530Disabling compacts and flushes for region at 1733692542530Disabling writes for close at 1733692542530Writing region close event to WAL at 1733692542530Closed at 1733692542530 2024-12-08T21:15:42,531 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:15:42,532 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733692542531"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733692542531"}]},"ts":"1733692542531"} 2024-12-08T21:15:42,533 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:15:42,534 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:15:42,534 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692542534"}]},"ts":"1733692542534"} 2024-12-08T21:15:42,536 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T21:15:42,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, ASSIGN}] 2024-12-08T21:15:42,538 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, ASSIGN 2024-12-08T21:15:42,539 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, ASSIGN; state=OFFLINE, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=false 2024-12-08T21:15:42,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:42,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, ASSIGN because future has completed 2024-12-08T21:15:42,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:15:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T21:15:42,854 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,854 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:15:42,854 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,854 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:15:42,854 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,855 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,856 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,858 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:42,858 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fdcd6355ad16589e9afb4e1e4d173eb columnFamilyName A 2024-12-08T21:15:42,858 DEBUG [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:42,859 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(327): Store=4fdcd6355ad16589e9afb4e1e4d173eb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:42,859 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,860 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:42,860 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fdcd6355ad16589e9afb4e1e4d173eb columnFamilyName B 2024-12-08T21:15:42,860 DEBUG [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:42,861 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(327): Store=4fdcd6355ad16589e9afb4e1e4d173eb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:42,861 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,862 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:42,862 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fdcd6355ad16589e9afb4e1e4d173eb columnFamilyName C 2024-12-08T21:15:42,862 DEBUG [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:42,863 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(327): Store=4fdcd6355ad16589e9afb4e1e4d173eb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:42,863 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,863 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,864 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,864 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,865 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,866 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,866 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:15:42,867 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,870 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:15:42,870 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 4fdcd6355ad16589e9afb4e1e4d173eb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74392541, jitterRate=0.10853524506092072}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:15:42,870 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:42,871 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 4fdcd6355ad16589e9afb4e1e4d173eb: Running coprocessor pre-open hook at 1733692542855Writing region info on filesystem at 1733692542855Initializing all the Stores at 1733692542856 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692542856Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692542856Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692542856Cleaning up temporary data from old regions at 1733692542866 (+10 ms)Running coprocessor post-open hooks at 1733692542870 (+4 ms)Region opened successfully at 1733692542871 (+1 ms) 2024-12-08T21:15:42,872 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., pid=89, masterSystemTime=1733692542847 2024-12-08T21:15:42,874 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,874 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:42,874 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=OPEN, openSeqNum=2, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:42,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:15:42,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-08T21:15:42,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 in 183 msec 2024-12-08T21:15:42,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-08T21:15:42,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, ASSIGN in 342 msec 2024-12-08T21:15:42,881 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:15:42,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692542881"}]},"ts":"1733692542881"} 2024-12-08T21:15:42,882 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T21:15:42,883 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:15:42,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-12-08T21:15:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-08T21:15:43,858 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:43,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:43,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:43,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:43,861 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15448242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:43,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:43,862 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0f7217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:43,863 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:43,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:43,866 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:15:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e653e208f502,45981,1733692443391 2024-12-08T21:15:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ebd3b43 2024-12-08T21:15:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:15:43,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:15:43,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:15:43,869 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:15:43,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T21:15:43,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742155_1331 (size=1097) 2024-12-08T21:15:44,281 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T21:15:44,282 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T21:15:44,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:15:44,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, REOPEN/MOVE}] 2024-12-08T21:15:44,288 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, REOPEN/MOVE 2024-12-08T21:15:44,289 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, REOPEN/MOVE because future has completed 2024-12-08T21:15:44,290 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:15:44,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:15:44,443 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,443 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:15:44,443 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 4fdcd6355ad16589e9afb4e1e4d173eb, disabling compactions & flushes 2024-12-08T21:15:44,443 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,443 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,443 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. after waiting 0 ms 2024-12-08T21:15:44,443 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,446 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T21:15:44,447 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,447 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 4fdcd6355ad16589e9afb4e1e4d173eb: Waiting for close lock at 1733692544443Running coprocessor pre-close hooks at 1733692544443Disabling compacts and flushes for region at 1733692544443Disabling writes for close at 1733692544443Writing region close event to WAL at 1733692544443Running coprocessor post-close hooks at 1733692544447 (+4 ms)Closed at 1733692544447 2024-12-08T21:15:44,447 WARN [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 4fdcd6355ad16589e9afb4e1e4d173eb to self. 2024-12-08T21:15:44,449 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,449 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=CLOSED 2024-12-08T21:15:44,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:15:44,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-08T21:15:44,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 in 162 msec 2024-12-08T21:15:44,455 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, REOPEN/MOVE; state=CLOSED, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=true 2024-12-08T21:15:44,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, REOPEN/MOVE because future has completed 2024-12-08T21:15:44,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:15:44,765 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,766 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:15:44,766 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,766 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:15:44,766 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,766 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,767 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,767 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:44,768 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fdcd6355ad16589e9afb4e1e4d173eb columnFamilyName A 2024-12-08T21:15:44,768 DEBUG [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:44,769 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(327): Store=4fdcd6355ad16589e9afb4e1e4d173eb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:44,769 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,770 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:44,770 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fdcd6355ad16589e9afb4e1e4d173eb columnFamilyName B 2024-12-08T21:15:44,770 DEBUG [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:44,770 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(327): Store=4fdcd6355ad16589e9afb4e1e4d173eb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:44,770 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,771 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:15:44,771 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fdcd6355ad16589e9afb4e1e4d173eb columnFamilyName C 2024-12-08T21:15:44,771 DEBUG [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:44,771 INFO [StoreOpener-4fdcd6355ad16589e9afb4e1e4d173eb-1 {}] regionserver.HStore(327): Store=4fdcd6355ad16589e9afb4e1e4d173eb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:15:44,771 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,771 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,772 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,773 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,773 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,773 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,774 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:15:44,774 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,775 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 4fdcd6355ad16589e9afb4e1e4d173eb; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70425875, jitterRate=0.04942731559276581}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:15:44,775 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,775 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 4fdcd6355ad16589e9afb4e1e4d173eb: Running coprocessor pre-open hook at 1733692544766Writing region info on filesystem at 1733692544766Initializing all the Stores at 1733692544766Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692544767 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692544767Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692544767Cleaning up temporary data from old regions at 1733692544773 (+6 ms)Running coprocessor post-open hooks at 1733692544775 (+2 ms)Region opened successfully at 1733692544775 2024-12-08T21:15:44,776 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., pid=94, masterSystemTime=1733692544763 2024-12-08T21:15:44,777 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,778 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,778 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=OPEN, openSeqNum=5, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:15:44,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-12-08T21:15:44,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 in 171 msec 2024-12-08T21:15:44,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-08T21:15:44,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, REOPEN/MOVE in 494 msec 2024-12-08T21:15:44,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-08T21:15:44,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-12-08T21:15:44,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-12-08T21:15:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-08T21:15:44,787 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:44,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,789 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cfe2553, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,795 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42329a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,798 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f4cec90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,799 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25af75f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ff344a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a9aac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,804 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609165c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,805 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@198a18dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,807 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a99353e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,808 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23b85b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,810 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc15348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,811 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32337e98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,813 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a210db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,814 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1398d22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,815 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ee2fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,817 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16c295e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,819 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1178e28f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,820 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1de7f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a84aab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:15:44,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:15:44,822 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:15:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:15:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:15:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422a5ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:15:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:15:44,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:15:44,824 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:15:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1181ea6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:15:44,825 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,825 DEBUG [Thread-1524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,825 DEBUG [Thread-1527 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,825 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,825 DEBUG [Thread-1522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,826 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,828 DEBUG [Thread-1516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-08T21:15:44,828 DEBUG [Thread-1518 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,829 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T21:15:44,830 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:44,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,838 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,838 DEBUG [Thread-1520 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:15:44,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:15:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:15:44,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:15:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:15:44,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:44,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:44,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:44,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:44,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:44,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:44,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085a4ae75e090b44caa85f7fb3a5dd3f6f_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692544836/Put/seqid=0 2024-12-08T21:15:44,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742156_1332 (size=12154) 2024-12-08T21:15:44,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:44,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412085a4ae75e090b44caa85f7fb3a5dd3f6f_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085a4ae75e090b44caa85f7fb3a5dd3f6f_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:44,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/113acad0b6f842199aaaaac60e35dff3, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:44,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/113acad0b6f842199aaaaac60e35dff3 is 175, key is test_row_0/A:col10/1733692544836/Put/seqid=0 2024-12-08T21:15:44,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:44,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692604884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692604885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692604888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:44,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742157_1333 (size=30955) 2024-12-08T21:15:44,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692544881, max timestamp=1733692544893Num uniques 859; 2024-12-08T21:15:44,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692544881, max timestamp=1733692544893Num uniques 859; 2024-12-08T21:15:44,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:44,905 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:44,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692544881, max timestamp=1733692544893Num uniques 859; 2024-12-08T21:15:44,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:15:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T21:15:44,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692604963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:44,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692604964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:44,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:44,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T21:15:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:44,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:44,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692604997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692604997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692604998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692605067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692605078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T21:15:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T21:15:45,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692605208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692605208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692605208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692605278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692605288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T21:15:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/113acad0b6f842199aaaaac60e35dff3 2024-12-08T21:15:45,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/df3c5599af0d4e8bbb6788f2ca2a1ab4 is 50, key is test_row_0/B:col10/1733692544836/Put/seqid=0 2024-12-08T21:15:45,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742158_1334 (size=12001) 2024-12-08T21:15:45,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T21:15:45,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T21:15:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692605528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692605528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692605528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692605587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T21:15:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692605597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:45,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:45,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T21:15:45,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:45,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/df3c5599af0d4e8bbb6788f2ca2a1ab4 2024-12-08T21:15:45,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/27d0f02f96b24a9a8eb2c3c36738f592 is 50, key is test_row_0/C:col10/1733692544836/Put/seqid=0 2024-12-08T21:15:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742159_1335 (size=12001) 2024-12-08T21:15:45,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/27d0f02f96b24a9a8eb2c3c36738f592 2024-12-08T21:15:45,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/113acad0b6f842199aaaaac60e35dff3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/113acad0b6f842199aaaaac60e35dff3 2024-12-08T21:15:45,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/113acad0b6f842199aaaaac60e35dff3, entries=150, sequenceid=16, filesize=30.2 K 2024-12-08T21:15:45,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/df3c5599af0d4e8bbb6788f2ca2a1ab4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/df3c5599af0d4e8bbb6788f2ca2a1ab4 2024-12-08T21:15:45,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/df3c5599af0d4e8bbb6788f2ca2a1ab4, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T21:15:45,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/27d0f02f96b24a9a8eb2c3c36738f592 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/27d0f02f96b24a9a8eb2c3c36738f592 2024-12-08T21:15:45,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/27d0f02f96b24a9a8eb2c3c36738f592, entries=150, sequenceid=16, filesize=11.7 K 2024-12-08T21:15:45,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 4fdcd6355ad16589e9afb4e1e4d173eb in 976ms, sequenceid=16, compaction requested=false 2024-12-08T21:15:45,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:45,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-08T21:15:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:45,905 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-08T21:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:45,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:15:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120825477c44b87b4ebbbac6bbcc4ccada02_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692544962/Put/seqid=0 2024-12-08T21:15:45,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742160_1336 (size=12154) 2024-12-08T21:15:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T21:15:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:46,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:46,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692606051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692606052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692606054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692606098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692606108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692606168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692606168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692606168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:46,328 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120825477c44b87b4ebbbac6bbcc4ccada02_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120825477c44b87b4ebbbac6bbcc4ccada02_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:46,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/216203f1268d4ddaa204a505acd5e4ec, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/216203f1268d4ddaa204a505acd5e4ec is 175, key is test_row_0/A:col10/1733692544962/Put/seqid=0 2024-12-08T21:15:46,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742161_1337 (size=30955) 2024-12-08T21:15:46,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692606378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692606378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692606378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,474 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:15:46,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692606688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692606688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692606688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:46,734 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/216203f1268d4ddaa204a505acd5e4ec 2024-12-08T21:15:46,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/ebad645e30fe4e65a6d065a3768a53f9 is 50, key is test_row_0/B:col10/1733692544962/Put/seqid=0 2024-12-08T21:15:46,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742162_1338 (size=12001) 2024-12-08T21:15:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T21:15:47,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692607117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:47,124 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:47,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692607128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:47,138 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:47,143 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/ebad645e30fe4e65a6d065a3768a53f9 2024-12-08T21:15:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/371586d6117b4878a4064806a82ddf17 is 50, key is test_row_0/C:col10/1733692544962/Put/seqid=0 2024-12-08T21:15:47,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742163_1339 (size=12001) 2024-12-08T21:15:47,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692607198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692607207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:47,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692607207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:47,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:47,553 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/371586d6117b4878a4064806a82ddf17 2024-12-08T21:15:47,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/216203f1268d4ddaa204a505acd5e4ec as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/216203f1268d4ddaa204a505acd5e4ec 2024-12-08T21:15:47,560 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/216203f1268d4ddaa204a505acd5e4ec, entries=150, sequenceid=52, filesize=30.2 K 2024-12-08T21:15:47,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/ebad645e30fe4e65a6d065a3768a53f9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/ebad645e30fe4e65a6d065a3768a53f9 2024-12-08T21:15:47,565 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/ebad645e30fe4e65a6d065a3768a53f9, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T21:15:47,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/371586d6117b4878a4064806a82ddf17 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/371586d6117b4878a4064806a82ddf17 2024-12-08T21:15:47,568 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/371586d6117b4878a4064806a82ddf17, entries=150, sequenceid=52, filesize=11.7 K 2024-12-08T21:15:47,569 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1664ms, sequenceid=52, compaction requested=false 2024-12-08T21:15:47,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:47,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:47,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-08T21:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-08T21:15:47,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-08T21:15:47,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7400 sec 2024-12-08T21:15:47,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.7460 sec 2024-12-08T21:15:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:48,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:15:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:48,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208277b9fc973be4142ba50dc1d0d812b43_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692546044/Put/seqid=0 2024-12-08T21:15:48,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742164_1340 (size=14594) 2024-12-08T21:15:48,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692608263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692608264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692608265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692608378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692608378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692608378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692608588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692608588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692608598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,632 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:48,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208277b9fc973be4142ba50dc1d0d812b43_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208277b9fc973be4142ba50dc1d0d812b43_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:48,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/1781c5df08b34b1cb1a3c9cb57954fab, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:48,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/1781c5df08b34b1cb1a3c9cb57954fab is 175, key is test_row_0/A:col10/1733692546044/Put/seqid=0 2024-12-08T21:15:48,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742165_1341 (size=39549) 2024-12-08T21:15:48,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692608897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692608898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:48,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692608908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:48,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-08T21:15:48,998 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:48,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-08T21:15:49,000 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T21:15:49,000 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:49,000 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:49,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/1781c5df08b34b1cb1a3c9cb57954fab 2024-12-08T21:15:49,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/a5e262f0c33548968a6ca55e8dbe373c is 50, key is test_row_0/B:col10/1733692546044/Put/seqid=0 2024-12-08T21:15:49,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742166_1342 (size=12001) 2024-12-08T21:15:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T21:15:49,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692609147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:49,150 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:49,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T21:15:49,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:49,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692609158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:49,164 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:49,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:49,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T21:15:49,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:49,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T21:15:49,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692609408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:49,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692609408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692609429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:49,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:49,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/a5e262f0c33548968a6ca55e8dbe373c 2024-12-08T21:15:49,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/6551ea8063ed4d998285380872c99d09 is 50, key is test_row_0/C:col10/1733692546044/Put/seqid=0 2024-12-08T21:15:49,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T21:15:49,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:49,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742167_1343 (size=12001) 2024-12-08T21:15:49,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T21:15:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T21:15:49,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T21:15:49,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:49,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:49,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/6551ea8063ed4d998285380872c99d09 2024-12-08T21:15:49,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/1781c5df08b34b1cb1a3c9cb57954fab as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/1781c5df08b34b1cb1a3c9cb57954fab 2024-12-08T21:15:49,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/1781c5df08b34b1cb1a3c9cb57954fab, entries=200, sequenceid=64, filesize=38.6 K 2024-12-08T21:15:49,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/a5e262f0c33548968a6ca55e8dbe373c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/a5e262f0c33548968a6ca55e8dbe373c 2024-12-08T21:15:49,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/a5e262f0c33548968a6ca55e8dbe373c, entries=150, sequenceid=64, filesize=11.7 K 2024-12-08T21:15:49,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/6551ea8063ed4d998285380872c99d09 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/6551ea8063ed4d998285380872c99d09 2024-12-08T21:15:49,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/6551ea8063ed4d998285380872c99d09, entries=150, sequenceid=64, filesize=11.7 K 2024-12-08T21:15:49,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1674ms, sequenceid=64, compaction requested=true 2024-12-08T21:15:49,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:49,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:49,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:49,895 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:49,896 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:49,897 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:49,897 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/A is initiating minor compaction (all files) 2024-12-08T21:15:49,897 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/A in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,897 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/113acad0b6f842199aaaaac60e35dff3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/216203f1268d4ddaa204a505acd5e4ec, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/1781c5df08b34b1cb1a3c9cb57954fab] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=99.1 K 2024-12-08T21:15:49,897 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,897 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/113acad0b6f842199aaaaac60e35dff3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/216203f1268d4ddaa204a505acd5e4ec, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/1781c5df08b34b1cb1a3c9cb57954fab] 2024-12-08T21:15:49,898 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 113acad0b6f842199aaaaac60e35dff3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733692544836 2024-12-08T21:15:49,898 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:49,899 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/B is initiating minor compaction (all files) 2024-12-08T21:15:49,899 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/B in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,899 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/df3c5599af0d4e8bbb6788f2ca2a1ab4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/ebad645e30fe4e65a6d065a3768a53f9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/a5e262f0c33548968a6ca55e8dbe373c] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.2 K 2024-12-08T21:15:49,899 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting df3c5599af0d4e8bbb6788f2ca2a1ab4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733692544836 2024-12-08T21:15:49,899 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216203f1268d4ddaa204a505acd5e4ec, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733692544923 2024-12-08T21:15:49,901 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ebad645e30fe4e65a6d065a3768a53f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733692544923 2024-12-08T21:15:49,901 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1781c5df08b34b1cb1a3c9cb57954fab, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733692546044 2024-12-08T21:15:49,901 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a5e262f0c33548968a6ca55e8dbe373c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733692546044 2024-12-08T21:15:49,910 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#B#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:49,911 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/0dc69a6b96d542ada59737379df6145b is 50, key is test_row_0/B:col10/1733692546044/Put/seqid=0 2024-12-08T21:15:49,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-08T21:15:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,916 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:15:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:49,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:49,937 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:49,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742168_1344 (size=12104) 2024-12-08T21:15:49,957 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412081b1acb0c75f7452196057e8cd1bb85fd_4fdcd6355ad16589e9afb4e1e4d173eb store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:49,959 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412081b1acb0c75f7452196057e8cd1bb85fd_4fdcd6355ad16589e9afb4e1e4d173eb, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:49,959 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081b1acb0c75f7452196057e8cd1bb85fd_4fdcd6355ad16589e9afb4e1e4d173eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:49,963 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/0dc69a6b96d542ada59737379df6145b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0dc69a6b96d542ada59737379df6145b 2024-12-08T21:15:49,970 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/B of 4fdcd6355ad16589e9afb4e1e4d173eb into 0dc69a6b96d542ada59737379df6145b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:49,970 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:49,970 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/B, priority=13, startTime=1733692549895; duration=0sec 2024-12-08T21:15:49,970 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:49,970 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:B 2024-12-08T21:15:49,970 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:49,971 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:49,971 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/C is initiating minor compaction (all files) 2024-12-08T21:15:49,971 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/C in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:49,971 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/27d0f02f96b24a9a8eb2c3c36738f592, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/371586d6117b4878a4064806a82ddf17, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/6551ea8063ed4d998285380872c99d09] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.2 K 2024-12-08T21:15:49,971 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 27d0f02f96b24a9a8eb2c3c36738f592, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733692544836 2024-12-08T21:15:49,972 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 371586d6117b4878a4064806a82ddf17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733692544923 2024-12-08T21:15:49,972 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6551ea8063ed4d998285380872c99d09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733692546044 2024-12-08T21:15:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120857a0b8e195e24160b30c61871f104012_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692548263/Put/seqid=0 2024-12-08T21:15:49,984 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:49,984 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/7580d391bb66491399c7ea0601211f18 is 50, key is test_row_0/C:col10/1733692546044/Put/seqid=0 2024-12-08T21:15:49,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742169_1345 (size=4469) 2024-12-08T21:15:50,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742170_1346 (size=12154) 2024-12-08T21:15:50,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742171_1347 (size=12104) 2024-12-08T21:15:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T21:15:50,397 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#A#compaction#300 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:50,397 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/18dffd59bd8546d397b545eb3ba6d31b is 175, key is test_row_0/A:col10/1733692546044/Put/seqid=0 2024-12-08T21:15:50,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:50,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742172_1348 (size=31058) 2024-12-08T21:15:50,431 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120857a0b8e195e24160b30c61871f104012_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120857a0b8e195e24160b30c61871f104012_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:50,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/4c9c0eb55d024aecbd58520052852b1b, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/4c9c0eb55d024aecbd58520052852b1b is 175, key is test_row_0/A:col10/1733692548263/Put/seqid=0 2024-12-08T21:15:50,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:50,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742173_1349 (size=30955) 2024-12-08T21:15:50,444 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/7580d391bb66491399c7ea0601211f18 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/7580d391bb66491399c7ea0601211f18 2024-12-08T21:15:50,448 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/C of 4fdcd6355ad16589e9afb4e1e4d173eb into 7580d391bb66491399c7ea0601211f18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:50,448 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:50,448 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/C, priority=13, startTime=1733692549896; duration=0sec 2024-12-08T21:15:50,448 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:50,448 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:C 2024-12-08T21:15:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692610453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692610454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692610455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692610568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692610578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692610578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692610778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692610789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692610789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:50,834 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/18dffd59bd8546d397b545eb3ba6d31b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/18dffd59bd8546d397b545eb3ba6d31b 2024-12-08T21:15:50,836 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/4c9c0eb55d024aecbd58520052852b1b 2024-12-08T21:15:50,838 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/A of 4fdcd6355ad16589e9afb4e1e4d173eb into 18dffd59bd8546d397b545eb3ba6d31b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:50,838 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:50,838 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/A, priority=13, startTime=1733692549895; duration=0sec 2024-12-08T21:15:50,838 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:50,838 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:A 2024-12-08T21:15:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/4135839c60644ed0bee2a9f0da25bd45 is 50, key is test_row_0/B:col10/1733692548263/Put/seqid=0 2024-12-08T21:15:50,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742174_1350 (size=12001) 2024-12-08T21:15:50,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/4135839c60644ed0bee2a9f0da25bd45 2024-12-08T21:15:50,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/0a061880279540219706b996c0abfbd2 is 50, key is test_row_0/C:col10/1733692548263/Put/seqid=0 2024-12-08T21:15:50,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742175_1351 (size=12001) 2024-12-08T21:15:51,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692611088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692611098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692611098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T21:15:51,274 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/0a061880279540219706b996c0abfbd2 2024-12-08T21:15:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/4c9c0eb55d024aecbd58520052852b1b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/4c9c0eb55d024aecbd58520052852b1b 2024-12-08T21:15:51,282 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/4c9c0eb55d024aecbd58520052852b1b, entries=150, sequenceid=84, filesize=30.2 K 2024-12-08T21:15:51,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/4135839c60644ed0bee2a9f0da25bd45 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4135839c60644ed0bee2a9f0da25bd45 2024-12-08T21:15:51,286 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4135839c60644ed0bee2a9f0da25bd45, entries=150, sequenceid=84, filesize=11.7 K 2024-12-08T21:15:51,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/0a061880279540219706b996c0abfbd2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a061880279540219706b996c0abfbd2 2024-12-08T21:15:51,291 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a061880279540219706b996c0abfbd2, entries=150, sequenceid=84, filesize=11.7 K 2024-12-08T21:15:51,292 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1376ms, sequenceid=84, compaction requested=false 2024-12-08T21:15:51,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:51,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:51,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-08T21:15:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-08T21:15:51,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-08T21:15:51,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2930 sec 2024-12-08T21:15:51,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.2980 sec 2024-12-08T21:15:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:51,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:15:51,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:51,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:51,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:51,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:51,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:51,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:51,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089071015deb5d44a18b4c8338754dcd11_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692550444/Put/seqid=0 2024-12-08T21:15:51,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742176_1352 (size=14594) 2024-12-08T21:15:51,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692611648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692611654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692611659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692611768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692611768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692611768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692611988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692611988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692611988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:52,012 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:52,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412089071015deb5d44a18b4c8338754dcd11_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089071015deb5d44a18b4c8338754dcd11_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:52,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/7480040113de4693873e1e98b33b7ab8, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:52,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/7480040113de4693873e1e98b33b7ab8 is 175, key is test_row_0/A:col10/1733692550444/Put/seqid=0 2024-12-08T21:15:52,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742177_1353 (size=39549) 2024-12-08T21:15:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692612299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692612299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692612299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:52,422 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/7480040113de4693873e1e98b33b7ab8 2024-12-08T21:15:52,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/dd60d2929ff54eceb2d00793888e1196 is 50, key is test_row_0/B:col10/1733692550444/Put/seqid=0 2024-12-08T21:15:52,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742178_1354 (size=12001) 2024-12-08T21:15:52,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692612818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:52,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692612818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:52,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692612818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:52,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:52,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/dd60d2929ff54eceb2d00793888e1196 2024-12-08T21:15:52,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/c55620accbd5486a916c2e9442d37b95 is 50, key is test_row_0/C:col10/1733692550444/Put/seqid=0 2024-12-08T21:15:52,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742179_1355 (size=12001) 2024-12-08T21:15:53,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-08T21:15:53,168 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:53,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-08T21:15:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T21:15:53,171 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:53,172 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:53,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:53,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692613187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692613187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,192 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:53,192 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:53,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:53,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/c55620accbd5486a916c2e9442d37b95 2024-12-08T21:15:53,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/7480040113de4693873e1e98b33b7ab8 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7480040113de4693873e1e98b33b7ab8 2024-12-08T21:15:53,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7480040113de4693873e1e98b33b7ab8, entries=200, sequenceid=100, filesize=38.6 K 2024-12-08T21:15:53,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/dd60d2929ff54eceb2d00793888e1196 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dd60d2929ff54eceb2d00793888e1196 2024-12-08T21:15:53,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dd60d2929ff54eceb2d00793888e1196, entries=150, sequenceid=100, filesize=11.7 K 2024-12-08T21:15:53,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/c55620accbd5486a916c2e9442d37b95 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c55620accbd5486a916c2e9442d37b95 2024-12-08T21:15:53,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c55620accbd5486a916c2e9442d37b95, entries=150, sequenceid=100, filesize=11.7 K 2024-12-08T21:15:53,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1677ms, sequenceid=100, compaction requested=true 2024-12-08T21:15:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:53,277 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:53,277 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:53,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T21:15:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:53,278 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:53,278 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/A is initiating minor compaction (all files) 2024-12-08T21:15:53,278 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/A in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:53,278 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/18dffd59bd8546d397b545eb3ba6d31b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/4c9c0eb55d024aecbd58520052852b1b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7480040113de4693873e1e98b33b7ab8] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=99.2 K 2024-12-08T21:15:53,278 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:53,278 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/18dffd59bd8546d397b545eb3ba6d31b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/4c9c0eb55d024aecbd58520052852b1b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7480040113de4693873e1e98b33b7ab8] 2024-12-08T21:15:53,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:53,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/B is initiating minor compaction (all files) 2024-12-08T21:15:53,279 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/B in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:53,279 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0dc69a6b96d542ada59737379df6145b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4135839c60644ed0bee2a9f0da25bd45, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dd60d2929ff54eceb2d00793888e1196] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.3 K 2024-12-08T21:15:53,279 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18dffd59bd8546d397b545eb3ba6d31b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733692546044 2024-12-08T21:15:53,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc69a6b96d542ada59737379df6145b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733692546044 2024-12-08T21:15:53,279 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c9c0eb55d024aecbd58520052852b1b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733692548249 2024-12-08T21:15:53,280 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4135839c60644ed0bee2a9f0da25bd45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733692548249 2024-12-08T21:15:53,280 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7480040113de4693873e1e98b33b7ab8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733692550444 2024-12-08T21:15:53,280 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting dd60d2929ff54eceb2d00793888e1196, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733692550444 2024-12-08T21:15:53,287 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:53,289 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:53,290 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/dfe89aff39e44641b50a536f376282ed is 50, key is test_row_0/B:col10/1733692550444/Put/seqid=0 2024-12-08T21:15:53,290 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208ef3327c9bce740a39045b63fc51d3666_4fdcd6355ad16589e9afb4e1e4d173eb store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:53,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208ef3327c9bce740a39045b63fc51d3666_4fdcd6355ad16589e9afb4e1e4d173eb, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:53,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ef3327c9bce740a39045b63fc51d3666_4fdcd6355ad16589e9afb4e1e4d173eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:53,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742180_1356 (size=12207) 2024-12-08T21:15:53,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742181_1357 (size=4469) 2024-12-08T21:15:53,303 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#A#compaction#308 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:53,303 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/7a109950d0fc4f21aa4978575a15e7a7 is 175, key is test_row_0/A:col10/1733692550444/Put/seqid=0 2024-12-08T21:15:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742182_1358 (size=31161) 2024-12-08T21:15:53,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-08T21:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:53,324 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T21:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:53,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:53,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081959c27137a149cabf681549b9e280f2_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692551640/Put/seqid=0 2024-12-08T21:15:53,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742183_1359 (size=12154) 2024-12-08T21:15:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T21:15:53,701 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/dfe89aff39e44641b50a536f376282ed as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dfe89aff39e44641b50a536f376282ed 2024-12-08T21:15:53,705 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/B of 4fdcd6355ad16589e9afb4e1e4d173eb into dfe89aff39e44641b50a536f376282ed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:53,705 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:53,705 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/B, priority=13, startTime=1733692553277; duration=0sec 2024-12-08T21:15:53,705 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:53,705 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:B 2024-12-08T21:15:53,705 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:53,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:53,706 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/C is initiating minor compaction (all files) 2024-12-08T21:15:53,706 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/C in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:53,706 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/7580d391bb66491399c7ea0601211f18, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a061880279540219706b996c0abfbd2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c55620accbd5486a916c2e9442d37b95] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.3 K 2024-12-08T21:15:53,707 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 7580d391bb66491399c7ea0601211f18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733692546044 2024-12-08T21:15:53,707 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a061880279540219706b996c0abfbd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733692548249 2024-12-08T21:15:53,707 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c55620accbd5486a916c2e9442d37b95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733692550444 2024-12-08T21:15:53,710 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/7a109950d0fc4f21aa4978575a15e7a7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7a109950d0fc4f21aa4978575a15e7a7 2024-12-08T21:15:53,714 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:53,714 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/b55c7688fb214a938fe9ac8e287bab3c is 50, key is test_row_0/C:col10/1733692550444/Put/seqid=0 2024-12-08T21:15:53,715 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/A of 4fdcd6355ad16589e9afb4e1e4d173eb into 7a109950d0fc4f21aa4978575a15e7a7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:53,715 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:53,715 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/A, priority=13, startTime=1733692553277; duration=0sec 2024-12-08T21:15:53,715 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:53,715 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:A 2024-12-08T21:15:53,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742184_1360 (size=12207) 2024-12-08T21:15:53,721 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/b55c7688fb214a938fe9ac8e287bab3c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b55c7688fb214a938fe9ac8e287bab3c 2024-12-08T21:15:53,726 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/C of 4fdcd6355ad16589e9afb4e1e4d173eb into b55c7688fb214a938fe9ac8e287bab3c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:53,726 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:53,726 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/C, priority=13, startTime=1733692553277; duration=0sec 2024-12-08T21:15:53,726 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:53,726 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:C 2024-12-08T21:15:53,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:53,736 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081959c27137a149cabf681549b9e280f2_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081959c27137a149cabf681549b9e280f2_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:53,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/c25f4363c8b44b55bb82f0e3240a2a7d, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:53,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/c25f4363c8b44b55bb82f0e3240a2a7d is 175, key is test_row_0/A:col10/1733692551640/Put/seqid=0 2024-12-08T21:15:53,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742185_1361 (size=30955) 2024-12-08T21:15:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T21:15:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:53,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:53,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692613847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:53,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692613848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:53,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692613849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692613958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692613958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:53,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692613958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,148 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/c25f4363c8b44b55bb82f0e3240a2a7d 2024-12-08T21:15:54,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/021802911ac8459fa311e5031aec96fc is 50, key is test_row_0/B:col10/1733692551640/Put/seqid=0 2024-12-08T21:15:54,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742186_1362 (size=12001) 2024-12-08T21:15:54,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692614168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692614168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692614168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T21:15:54,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692614478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692614478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692614478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,570 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/021802911ac8459fa311e5031aec96fc 2024-12-08T21:15:54,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/c1c2c50c1a4244f080a1b2262be6cf14 is 50, key is test_row_0/C:col10/1733692551640/Put/seqid=0 2024-12-08T21:15:54,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742187_1363 (size=12001) 2024-12-08T21:15:54,985 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/c1c2c50c1a4244f080a1b2262be6cf14 2024-12-08T21:15:54,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/c25f4363c8b44b55bb82f0e3240a2a7d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/c25f4363c8b44b55bb82f0e3240a2a7d 2024-12-08T21:15:54,991 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/c25f4363c8b44b55bb82f0e3240a2a7d, entries=150, sequenceid=119, filesize=30.2 K 2024-12-08T21:15:54,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/021802911ac8459fa311e5031aec96fc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/021802911ac8459fa311e5031aec96fc 2024-12-08T21:15:54,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692614988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692614988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:54,996 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/021802911ac8459fa311e5031aec96fc, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T21:15:54,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/c1c2c50c1a4244f080a1b2262be6cf14 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c1c2c50c1a4244f080a1b2262be6cf14 2024-12-08T21:15:55,001 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c1c2c50c1a4244f080a1b2262be6cf14, entries=150, sequenceid=119, filesize=11.7 K 2024-12-08T21:15:55,002 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1678ms, sequenceid=119, compaction requested=false 2024-12-08T21:15:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-08T21:15:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-08T21:15:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:55,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T21:15:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:55,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:55,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-08T21:15:55,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-12-08T21:15:55,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8360 sec 2024-12-08T21:15:55,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120828118d2d380242aaacfa2682ebb7a695_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692555003/Put/seqid=0 2024-12-08T21:15:55,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742188_1364 (size=14744) 2024-12-08T21:15:55,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:55,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120828118d2d380242aaacfa2682ebb7a695_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120828118d2d380242aaacfa2682ebb7a695_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:55,017 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/3c96414b259a4187b706fe29ab0eaf65, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:55,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/3c96414b259a4187b706fe29ab0eaf65 is 175, key is test_row_0/A:col10/1733692555003/Put/seqid=0 2024-12-08T21:15:55,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742189_1365 (size=39699) 2024-12-08T21:15:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692615092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:55,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692615198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-08T21:15:55,318 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:55,318 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-08T21:15:55,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T21:15:55,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:55,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:55,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692615408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:55,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/3c96414b259a4187b706fe29ab0eaf65 2024-12-08T21:15:55,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/4d71b2b80d5a437ab8810861325e4ff3 is 50, key is test_row_0/B:col10/1733692555003/Put/seqid=0 2024-12-08T21:15:55,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T21:15:55,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742190_1366 (size=12101) 2024-12-08T21:15:55,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T21:15:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:55,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T21:15:55,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T21:15:55,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692615718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:55,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:55,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T21:15:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/4d71b2b80d5a437ab8810861325e4ff3 2024-12-08T21:15:55,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/eac733badfa34dc8b8888e4fa191ce79 is 50, key is test_row_0/C:col10/1733692555003/Put/seqid=0 2024-12-08T21:15:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742191_1367 (size=12101) 2024-12-08T21:15:55,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T21:15:55,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:55,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:55,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T21:15:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692616007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:56,009 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:56,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:56,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692616008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:56,013 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:56,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T21:15:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:56,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692616227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:56,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:56,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T21:15:56,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:56,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:56,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:15:56,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/eac733badfa34dc8b8888e4fa191ce79 2024-12-08T21:15:56,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/3c96414b259a4187b706fe29ab0eaf65 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3c96414b259a4187b706fe29ab0eaf65 2024-12-08T21:15:56,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3c96414b259a4187b706fe29ab0eaf65, entries=200, sequenceid=136, filesize=38.8 K 2024-12-08T21:15:56,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/4d71b2b80d5a437ab8810861325e4ff3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4d71b2b80d5a437ab8810861325e4ff3 2024-12-08T21:15:56,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4d71b2b80d5a437ab8810861325e4ff3, entries=150, sequenceid=136, filesize=11.8 K 2024-12-08T21:15:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/eac733badfa34dc8b8888e4fa191ce79 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/eac733badfa34dc8b8888e4fa191ce79 2024-12-08T21:15:56,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/eac733badfa34dc8b8888e4fa191ce79, entries=150, sequenceid=136, filesize=11.8 K 2024-12-08T21:15:56,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1251ms, sequenceid=136, compaction requested=true 2024-12-08T21:15:56,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:56,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:56,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:56,255 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:56,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:56,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:56,255 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:56,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:56,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:56,256 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:56,256 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:56,256 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/A is initiating minor compaction (all files) 2024-12-08T21:15:56,256 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/B is initiating minor compaction (all files) 2024-12-08T21:15:56,256 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/B in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,256 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/A in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,256 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7a109950d0fc4f21aa4978575a15e7a7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/c25f4363c8b44b55bb82f0e3240a2a7d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3c96414b259a4187b706fe29ab0eaf65] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=99.4 K 2024-12-08T21:15:56,256 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dfe89aff39e44641b50a536f376282ed, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/021802911ac8459fa311e5031aec96fc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4d71b2b80d5a437ab8810861325e4ff3] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.5 K 2024-12-08T21:15:56,256 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,256 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7a109950d0fc4f21aa4978575a15e7a7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/c25f4363c8b44b55bb82f0e3240a2a7d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3c96414b259a4187b706fe29ab0eaf65] 2024-12-08T21:15:56,257 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe89aff39e44641b50a536f376282ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733692550444 2024-12-08T21:15:56,257 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a109950d0fc4f21aa4978575a15e7a7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733692550444 2024-12-08T21:15:56,257 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 021802911ac8459fa311e5031aec96fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733692551640 2024-12-08T21:15:56,257 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c25f4363c8b44b55bb82f0e3240a2a7d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733692551640 2024-12-08T21:15:56,257 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d71b2b80d5a437ab8810861325e4ff3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733692553837 2024-12-08T21:15:56,257 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c96414b259a4187b706fe29ab0eaf65, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733692553837 2024-12-08T21:15:56,263 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#B#compaction#317 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:56,264 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/071114f76a224b4cb268ccc2523b5880 is 50, key is test_row_0/B:col10/1733692555003/Put/seqid=0 2024-12-08T21:15:56,265 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:56,269 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412081c42b44b8a1e4685b1e26b77b829e300_4fdcd6355ad16589e9afb4e1e4d173eb store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:56,270 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412081c42b44b8a1e4685b1e26b77b829e300_4fdcd6355ad16589e9afb4e1e4d173eb, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:56,270 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081c42b44b8a1e4685b1e26b77b829e300_4fdcd6355ad16589e9afb4e1e4d173eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:56,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742192_1368 (size=12409) 2024-12-08T21:15:56,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742193_1369 (size=4469) 2024-12-08T21:15:56,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-08T21:15:56,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,389 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T21:15:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:56,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:56,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bcd5f9f2cc0a4d969e3c42a65c552943_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692555062/Put/seqid=0 2024-12-08T21:15:56,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742194_1370 (size=12304) 2024-12-08T21:15:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T21:15:56,682 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:56,683 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/8d0302f9e2054d3b93dc98fbb7b4b999 is 175, key is test_row_0/A:col10/1733692555003/Put/seqid=0 2024-12-08T21:15:56,683 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/071114f76a224b4cb268ccc2523b5880 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/071114f76a224b4cb268ccc2523b5880 2024-12-08T21:15:56,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742195_1371 (size=31363) 2024-12-08T21:15:56,688 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/B of 4fdcd6355ad16589e9afb4e1e4d173eb into 071114f76a224b4cb268ccc2523b5880(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:56,688 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:56,689 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/B, priority=13, startTime=1733692556255; duration=0sec 2024-12-08T21:15:56,689 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:56,689 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:B 2024-12-08T21:15:56,689 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:56,691 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/8d0302f9e2054d3b93dc98fbb7b4b999 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/8d0302f9e2054d3b93dc98fbb7b4b999 2024-12-08T21:15:56,691 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:56,691 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/C is initiating minor compaction (all files) 2024-12-08T21:15:56,691 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/C in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:56,691 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b55c7688fb214a938fe9ac8e287bab3c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c1c2c50c1a4244f080a1b2262be6cf14, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/eac733badfa34dc8b8888e4fa191ce79] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.5 K 2024-12-08T21:15:56,691 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b55c7688fb214a938fe9ac8e287bab3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1733692550444 2024-12-08T21:15:56,692 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c2c50c1a4244f080a1b2262be6cf14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733692551640 2024-12-08T21:15:56,692 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting eac733badfa34dc8b8888e4fa191ce79, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733692553837 2024-12-08T21:15:56,696 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/A of 4fdcd6355ad16589e9afb4e1e4d173eb into 8d0302f9e2054d3b93dc98fbb7b4b999(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:56,696 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:56,696 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/A, priority=13, startTime=1733692556255; duration=0sec 2024-12-08T21:15:56,696 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:56,696 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:A 2024-12-08T21:15:56,705 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#C#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:56,705 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/8ac8c206b9d6496d95e8e95d5bd61d1d is 50, key is test_row_0/C:col10/1733692555003/Put/seqid=0 2024-12-08T21:15:56,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742196_1372 (size=12409) 2024-12-08T21:15:56,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:56,803 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bcd5f9f2cc0a4d969e3c42a65c552943_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bcd5f9f2cc0a4d969e3c42a65c552943_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:56,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/10c1edf86013480bbb6367af0c95f96b, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:56,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/10c1edf86013480bbb6367af0c95f96b is 175, key is test_row_0/A:col10/1733692555062/Put/seqid=0 2024-12-08T21:15:56,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742197_1373 (size=31105) 2024-12-08T21:15:56,809 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/10c1edf86013480bbb6367af0c95f96b 2024-12-08T21:15:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/92aff8757aa34f7e92d620ea6b245c36 is 50, key is test_row_0/B:col10/1733692555062/Put/seqid=0 2024-12-08T21:15:56,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742198_1374 (size=12151) 2024-12-08T21:15:57,113 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/8ac8c206b9d6496d95e8e95d5bd61d1d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/8ac8c206b9d6496d95e8e95d5bd61d1d 2024-12-08T21:15:57,117 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/C of 4fdcd6355ad16589e9afb4e1e4d173eb into 8ac8c206b9d6496d95e8e95d5bd61d1d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:15:57,117 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:57,117 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/C, priority=13, startTime=1733692556255; duration=0sec 2024-12-08T21:15:57,117 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:15:57,117 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:C 2024-12-08T21:15:57,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/92aff8757aa34f7e92d620ea6b245c36 2024-12-08T21:15:57,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/0a38410649484e88844b0fc081ee6212 is 50, key is test_row_0/C:col10/1733692555062/Put/seqid=0 2024-12-08T21:15:57,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742199_1375 (size=12151) 2024-12-08T21:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:57,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:15:57,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692617275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:57,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692617388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T21:15:57,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:57,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692617597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:57,631 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/0a38410649484e88844b0fc081ee6212 2024-12-08T21:15:57,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/10c1edf86013480bbb6367af0c95f96b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/10c1edf86013480bbb6367af0c95f96b 2024-12-08T21:15:57,638 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/10c1edf86013480bbb6367af0c95f96b, entries=150, sequenceid=154, filesize=30.4 K 2024-12-08T21:15:57,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/92aff8757aa34f7e92d620ea6b245c36 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/92aff8757aa34f7e92d620ea6b245c36 2024-12-08T21:15:57,642 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/92aff8757aa34f7e92d620ea6b245c36, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T21:15:57,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/0a38410649484e88844b0fc081ee6212 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a38410649484e88844b0fc081ee6212 2024-12-08T21:15:57,647 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a38410649484e88844b0fc081ee6212, entries=150, sequenceid=154, filesize=11.9 K 2024-12-08T21:15:57,648 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1258ms, sequenceid=154, compaction requested=false 2024-12-08T21:15:57,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:57,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:57,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-08T21:15:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-08T21:15:57,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-08T21:15:57,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3290 sec 2024-12-08T21:15:57,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3340 sec 2024-12-08T21:15:57,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:57,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T21:15:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:57,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:57,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:57,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086beca0999d4e4c70836d8e794cf0696c_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692557274/Put/seqid=0 2024-12-08T21:15:57,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742200_1376 (size=14794) 2024-12-08T21:15:57,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692617969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:57,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692618028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:58,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:58,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692618028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:58,034 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:58,034 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:15:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:58,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:58,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692618078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692618288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:58,320 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:15:58,323 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086beca0999d4e4c70836d8e794cf0696c_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086beca0999d4e4c70836d8e794cf0696c_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:15:58,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/687a315eb470449893865e370dc4bad1, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:58,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/687a315eb470449893865e370dc4bad1 is 175, key is test_row_0/A:col10/1733692557274/Put/seqid=0 2024-12-08T21:15:58,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742201_1377 (size=39749) 2024-12-08T21:15:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692618597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:58,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:58,727 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/687a315eb470449893865e370dc4bad1 2024-12-08T21:15:58,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/0f7bf001c8514507b516426cc24ab028 is 50, key is test_row_0/B:col10/1733692557274/Put/seqid=0 2024-12-08T21:15:58,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742202_1378 (size=12151) 2024-12-08T21:15:59,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:15:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692619107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:15:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:15:59,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:15:59,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/0f7bf001c8514507b516426cc24ab028 2024-12-08T21:15:59,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/9c01d19ea339446e9689c6934ab77b56 is 50, key is test_row_0/C:col10/1733692557274/Put/seqid=0 2024-12-08T21:15:59,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742203_1379 (size=12151) 2024-12-08T21:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-08T21:15:59,478 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:15:59,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:15:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-08T21:15:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T21:15:59,480 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:15:59,481 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:15:59,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:15:59,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/9c01d19ea339446e9689c6934ab77b56 2024-12-08T21:15:59,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/687a315eb470449893865e370dc4bad1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/687a315eb470449893865e370dc4bad1 2024-12-08T21:15:59,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/687a315eb470449893865e370dc4bad1, entries=200, sequenceid=172, filesize=38.8 K 2024-12-08T21:15:59,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/0f7bf001c8514507b516426cc24ab028 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0f7bf001c8514507b516426cc24ab028 2024-12-08T21:15:59,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0f7bf001c8514507b516426cc24ab028, entries=150, sequenceid=172, filesize=11.9 K 2024-12-08T21:15:59,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/9c01d19ea339446e9689c6934ab77b56 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/9c01d19ea339446e9689c6934ab77b56 2024-12-08T21:15:59,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/9c01d19ea339446e9689c6934ab77b56, entries=150, sequenceid=172, filesize=11.9 K 2024-12-08T21:15:59,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1656ms, sequenceid=172, compaction requested=true 2024-12-08T21:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-08T21:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:15:59,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-08T21:15:59,567 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:59,567 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:15:59,568 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:59,568 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/C is initiating minor compaction (all files) 2024-12-08T21:15:59,568 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/C in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:59,568 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/8ac8c206b9d6496d95e8e95d5bd61d1d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a38410649484e88844b0fc081ee6212, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/9c01d19ea339446e9689c6934ab77b56] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.9 K 2024-12-08T21:15:59,568 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:15:59,569 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/A is initiating minor compaction (all files) 2024-12-08T21:15:59,569 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/A in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:59,569 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/8d0302f9e2054d3b93dc98fbb7b4b999, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/10c1edf86013480bbb6367af0c95f96b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/687a315eb470449893865e370dc4bad1] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=99.8 K 2024-12-08T21:15:59,569 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:59,569 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/8d0302f9e2054d3b93dc98fbb7b4b999, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/10c1edf86013480bbb6367af0c95f96b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/687a315eb470449893865e370dc4bad1] 2024-12-08T21:15:59,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ac8c206b9d6496d95e8e95d5bd61d1d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733692553837 2024-12-08T21:15:59,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d0302f9e2054d3b93dc98fbb7b4b999, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733692553837 2024-12-08T21:15:59,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a38410649484e88844b0fc081ee6212, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733692555062 2024-12-08T21:15:59,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10c1edf86013480bbb6367af0c95f96b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733692555062 2024-12-08T21:15:59,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c01d19ea339446e9689c6934ab77b56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733692557262 2024-12-08T21:15:59,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 687a315eb470449893865e370dc4bad1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733692557262 2024-12-08T21:15:59,579 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:59,579 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:59,579 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/1c028ef03fab4330988558c8247dcaec is 50, key is test_row_0/C:col10/1733692557274/Put/seqid=0 2024-12-08T21:15:59,581 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412087b9f37de021f41cdbedbf18f7a485223_4fdcd6355ad16589e9afb4e1e4d173eb store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:59,583 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412087b9f37de021f41cdbedbf18f7a485223_4fdcd6355ad16589e9afb4e1e4d173eb, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:59,583 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412087b9f37de021f41cdbedbf18f7a485223_4fdcd6355ad16589e9afb4e1e4d173eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:15:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T21:15:59,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742205_1381 (size=12561) 2024-12-08T21:15:59,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742204_1380 (size=4469) 2024-12-08T21:15:59,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-08T21:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:15:59,633 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T21:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:15:59,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:15:59,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120844f3de91a53e496bb1d12d39f1318af1_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692557960/Put/seqid=0 2024-12-08T21:15:59,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742206_1382 (size=12304) 2024-12-08T21:15:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T21:15:59,995 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/1c028ef03fab4330988558c8247dcaec as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/1c028ef03fab4330988558c8247dcaec 2024-12-08T21:15:59,998 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:15:59,999 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/efa972211f954b9f840c766565b8fd19 is 175, key is test_row_0/A:col10/1733692557274/Put/seqid=0 2024-12-08T21:16:00,000 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/C of 4fdcd6355ad16589e9afb4e1e4d173eb into 1c028ef03fab4330988558c8247dcaec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:00,000 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:00,000 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/C, priority=13, startTime=1733692559566; duration=0sec 2024-12-08T21:16:00,001 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:00,001 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:C 2024-12-08T21:16:00,001 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:00,002 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:00,002 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/B is initiating minor compaction (all files) 2024-12-08T21:16:00,002 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/B in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:00,002 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/071114f76a224b4cb268ccc2523b5880, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/92aff8757aa34f7e92d620ea6b245c36, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0f7bf001c8514507b516426cc24ab028] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=35.9 K 2024-12-08T21:16:00,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742207_1383 (size=31515) 2024-12-08T21:16:00,005 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 071114f76a224b4cb268ccc2523b5880, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733692553837 2024-12-08T21:16:00,005 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 92aff8757aa34f7e92d620ea6b245c36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1733692555062 2024-12-08T21:16:00,005 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7bf001c8514507b516426cc24ab028, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733692557262 2024-12-08T21:16:00,010 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#B#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:00,011 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/873c041d97284d05ad418720b1877a99 is 50, key is test_row_0/B:col10/1733692557274/Put/seqid=0 2024-12-08T21:16:00,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742208_1384 (size=12561) 2024-12-08T21:16:00,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:00,047 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120844f3de91a53e496bb1d12d39f1318af1_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120844f3de91a53e496bb1d12d39f1318af1_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/47a6d7295c674a60a765fd4953a7dee5, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:00,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/47a6d7295c674a60a765fd4953a7dee5 is 175, key is test_row_0/A:col10/1733692557960/Put/seqid=0 2024-12-08T21:16:00,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742209_1385 (size=31105) 2024-12-08T21:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T21:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:00,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692620207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:00,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692620318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:00,409 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/efa972211f954b9f840c766565b8fd19 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/efa972211f954b9f840c766565b8fd19 2024-12-08T21:16:00,414 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/A of 4fdcd6355ad16589e9afb4e1e4d173eb into efa972211f954b9f840c766565b8fd19(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:00,414 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:00,414 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/A, priority=13, startTime=1733692559566; duration=0sec 2024-12-08T21:16:00,414 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:00,414 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:A 2024-12-08T21:16:00,429 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/873c041d97284d05ad418720b1877a99 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/873c041d97284d05ad418720b1877a99 2024-12-08T21:16:00,435 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/B of 4fdcd6355ad16589e9afb4e1e4d173eb into 873c041d97284d05ad418720b1877a99(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:00,435 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:00,435 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/B, priority=13, startTime=1733692559566; duration=0sec 2024-12-08T21:16:00,435 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:00,435 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:B 2024-12-08T21:16:00,455 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/47a6d7295c674a60a765fd4953a7dee5 2024-12-08T21:16:00,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/93dc48e9dfb44ba5a25f549cd7068d77 is 50, key is test_row_0/B:col10/1733692557960/Put/seqid=0 2024-12-08T21:16:00,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742210_1386 (size=12151) 2024-12-08T21:16:00,485 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/93dc48e9dfb44ba5a25f549cd7068d77 2024-12-08T21:16:00,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/82e46bfb829a4c21b23076c15c606902 is 50, key is test_row_0/C:col10/1733692557960/Put/seqid=0 2024-12-08T21:16:00,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742211_1387 (size=12151) 2024-12-08T21:16:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692620528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T21:16:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692620838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:00,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:00,928 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/82e46bfb829a4c21b23076c15c606902 2024-12-08T21:16:00,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/47a6d7295c674a60a765fd4953a7dee5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/47a6d7295c674a60a765fd4953a7dee5 2024-12-08T21:16:00,940 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/47a6d7295c674a60a765fd4953a7dee5, entries=150, sequenceid=189, filesize=30.4 K 2024-12-08T21:16:00,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/93dc48e9dfb44ba5a25f549cd7068d77 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/93dc48e9dfb44ba5a25f549cd7068d77 2024-12-08T21:16:00,951 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/93dc48e9dfb44ba5a25f549cd7068d77, entries=150, sequenceid=189, filesize=11.9 K 2024-12-08T21:16:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/82e46bfb829a4c21b23076c15c606902 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/82e46bfb829a4c21b23076c15c606902 2024-12-08T21:16:00,965 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/82e46bfb829a4c21b23076c15c606902, entries=150, sequenceid=189, filesize=11.9 K 2024-12-08T21:16:00,966 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1333ms, sequenceid=189, compaction requested=false 2024-12-08T21:16:00,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-08T21:16:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-08T21:16:00,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-08T21:16:00,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4870 sec 2024-12-08T21:16:00,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4950 sec 2024-12-08T21:16:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:01,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T21:16:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:16:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:16:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:16:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:01,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208042d5687463146e5bc55bbf9a6c979f4_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692560205/Put/seqid=0 2024-12-08T21:16:01,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742212_1388 (size=14794) 2024-12-08T21:16:01,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:01,432 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208042d5687463146e5bc55bbf9a6c979f4_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208042d5687463146e5bc55bbf9a6c979f4_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:01,433 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/0f862a46e3f14fb883f8154b9a13597a, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:01,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/0f862a46e3f14fb883f8154b9a13597a is 175, key is test_row_0/A:col10/1733692560205/Put/seqid=0 2024-12-08T21:16:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742213_1389 (size=39749) 2024-12-08T21:16:01,458 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/0f862a46e3f14fb883f8154b9a13597a 2024-12-08T21:16:01,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/bec010ff3315427799db2ca9d31efea7 is 50, key is test_row_0/B:col10/1733692560205/Put/seqid=0 2024-12-08T21:16:01,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742214_1390 (size=12151) 2024-12-08T21:16:01,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692621498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:01,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692621608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-08T21:16:01,628 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:01,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-08T21:16:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T21:16:01,631 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:01,631 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:01,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T21:16:01,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T21:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:01,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:01,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:01,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692621828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:01,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:01,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/bec010ff3315427799db2ca9d31efea7 2024-12-08T21:16:01,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/251eb7ad9855493d8b7f678df67e5176 is 50, key is test_row_0/C:col10/1733692560205/Put/seqid=0 2024-12-08T21:16:01,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742215_1391 (size=12151) 2024-12-08T21:16:01,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T21:16:01,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:01,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:01,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:01,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:01,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T21:16:02,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37876 deadline: 1733692622048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:02,050 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:02,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37906 deadline: 1733692622068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:02,073 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:02,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T21:16:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:02,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:02,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:02,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692622148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:02,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T21:16:02,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:02,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:02,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T21:16:02,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/251eb7ad9855493d8b7f678df67e5176 2024-12-08T21:16:02,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/0f862a46e3f14fb883f8154b9a13597a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/0f862a46e3f14fb883f8154b9a13597a 2024-12-08T21:16:02,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/0f862a46e3f14fb883f8154b9a13597a, entries=200, sequenceid=208, filesize=38.8 K 2024-12-08T21:16:02,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/bec010ff3315427799db2ca9d31efea7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/bec010ff3315427799db2ca9d31efea7 2024-12-08T21:16:02,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/bec010ff3315427799db2ca9d31efea7, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T21:16:02,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/251eb7ad9855493d8b7f678df67e5176 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/251eb7ad9855493d8b7f678df67e5176 2024-12-08T21:16:02,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/251eb7ad9855493d8b7f678df67e5176, entries=150, sequenceid=208, filesize=11.9 K 2024-12-08T21:16:02,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4fdcd6355ad16589e9afb4e1e4d173eb in 989ms, sequenceid=208, compaction requested=true 2024-12-08T21:16:02,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:02,354 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:02,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:02,355 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:02,355 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/A is initiating minor compaction (all files) 2024-12-08T21:16:02,355 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/A in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,355 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/efa972211f954b9f840c766565b8fd19, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/47a6d7295c674a60a765fd4953a7dee5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/0f862a46e3f14fb883f8154b9a13597a] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=100.0 K 2024-12-08T21:16:02,355 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,355 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/efa972211f954b9f840c766565b8fd19, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/47a6d7295c674a60a765fd4953a7dee5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/0f862a46e3f14fb883f8154b9a13597a] 2024-12-08T21:16:02,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:02,355 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:02,355 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting efa972211f954b9f840c766565b8fd19, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733692557262 2024-12-08T21:16:02,357 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47a6d7295c674a60a765fd4953a7dee5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733692557960 2024-12-08T21:16:02,358 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:02,358 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/B is initiating minor compaction (all files) 2024-12-08T21:16:02,358 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/B in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,358 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/873c041d97284d05ad418720b1877a99, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/93dc48e9dfb44ba5a25f549cd7068d77, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/bec010ff3315427799db2ca9d31efea7] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=36.0 K 2024-12-08T21:16:02,358 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f862a46e3f14fb883f8154b9a13597a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733692560187 2024-12-08T21:16:02,359 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 873c041d97284d05ad418720b1877a99, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733692557262 2024-12-08T21:16:02,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:02,359 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 93dc48e9dfb44ba5a25f549cd7068d77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733692557960 2024-12-08T21:16:02,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:02,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:02,360 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting bec010ff3315427799db2ca9d31efea7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733692560203 2024-12-08T21:16:02,377 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:02,381 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412082dfec3d6505141afb5a5ce82a77b58b8_4fdcd6355ad16589e9afb4e1e4d173eb store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:02,381 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#B#compaction#336 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:02,382 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/741686afbede489a8c62fe988a78531b is 50, key is test_row_0/B:col10/1733692560205/Put/seqid=0 2024-12-08T21:16:02,383 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412082dfec3d6505141afb5a5ce82a77b58b8_4fdcd6355ad16589e9afb4e1e4d173eb, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:02,383 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082dfec3d6505141afb5a5ce82a77b58b8_4fdcd6355ad16589e9afb4e1e4d173eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:02,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742216_1392 (size=12663) 2024-12-08T21:16:02,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742217_1393 (size=4469) 2024-12-08T21:16:02,403 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#A#compaction#335 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:02,404 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/3938e7f0a9b94322ad397b680a80c8c7 is 175, key is test_row_0/A:col10/1733692560205/Put/seqid=0 2024-12-08T21:16:02,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-08T21:16:02,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,405 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T21:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:02,409 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/741686afbede489a8c62fe988a78531b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/741686afbede489a8c62fe988a78531b 2024-12-08T21:16:02,414 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/B of 4fdcd6355ad16589e9afb4e1e4d173eb into 741686afbede489a8c62fe988a78531b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:02,415 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:02,415 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/B, priority=13, startTime=1733692562355; duration=0sec 2024-12-08T21:16:02,415 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:02,415 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:B 2024-12-08T21:16:02,415 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:02,415 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:02,415 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/C is initiating minor compaction (all files) 2024-12-08T21:16:02,416 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/C in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:02,416 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/1c028ef03fab4330988558c8247dcaec, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/82e46bfb829a4c21b23076c15c606902, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/251eb7ad9855493d8b7f678df67e5176] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=36.0 K 2024-12-08T21:16:02,416 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c028ef03fab4330988558c8247dcaec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1733692557262 2024-12-08T21:16:02,416 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 82e46bfb829a4c21b23076c15c606902, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733692557960 2024-12-08T21:16:02,417 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 251eb7ad9855493d8b7f678df67e5176, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733692560203 2024-12-08T21:16:02,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742218_1394 (size=31617) 2024-12-08T21:16:02,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120858f062026b2b49b294b3ec15122a6abd_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692561488/Put/seqid=0 2024-12-08T21:16:02,435 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/3938e7f0a9b94322ad397b680a80c8c7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3938e7f0a9b94322ad397b680a80c8c7 2024-12-08T21:16:02,439 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#C#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:02,439 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/b6df09aff09949168133a0301179b139 is 50, key is test_row_0/C:col10/1733692560205/Put/seqid=0 2024-12-08T21:16:02,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:16:02,447 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/A of 4fdcd6355ad16589e9afb4e1e4d173eb into 3938e7f0a9b94322ad397b680a80c8c7(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:02,447 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:02,447 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/A, priority=13, startTime=1733692562354; duration=0sec 2024-12-08T21:16:02,448 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:02,448 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:A 2024-12-08T21:16:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742219_1395 (size=12304) 2024-12-08T21:16:02,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742220_1396 (size=12663) 2024-12-08T21:16:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:02,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:02,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692622761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:02,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T21:16:02,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:02,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120858f062026b2b49b294b3ec15122a6abd_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120858f062026b2b49b294b3ec15122a6abd_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:02,858 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/b6df09aff09949168133a0301179b139 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b6df09aff09949168133a0301179b139 2024-12-08T21:16:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/b322e4805c0d4dbba2b007c1cd2c3442, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:02,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/b322e4805c0d4dbba2b007c1cd2c3442 is 175, key is test_row_0/A:col10/1733692561488/Put/seqid=0 2024-12-08T21:16:02,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742221_1397 (size=31105) 2024-12-08T21:16:02,862 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/C of 4fdcd6355ad16589e9afb4e1e4d173eb into b6df09aff09949168133a0301179b139(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:02,862 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:02,862 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/C, priority=13, startTime=1733692562360; duration=0sec 2024-12-08T21:16:02,862 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/b322e4805c0d4dbba2b007c1cd2c3442 2024-12-08T21:16:02,862 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:02,862 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:C 2024-12-08T21:16:02,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/e66c1a87c89f4814891223975a6e7906 is 50, key is test_row_0/B:col10/1733692561488/Put/seqid=0 2024-12-08T21:16:02,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742222_1398 (size=12151) 2024-12-08T21:16:02,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692622878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692623097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:03,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37922 deadline: 1733692623208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:03,209 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:03,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37834 deadline: 1733692623258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:03,262 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 4fdcd6355ad16589e9afb4e1e4d173eb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18299 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:03,273 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/e66c1a87c89f4814891223975a6e7906 2024-12-08T21:16:03,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/f6c6051550de49c69f0398324d5e3bf8 is 50, key is test_row_0/C:col10/1733692561488/Put/seqid=0 2024-12-08T21:16:03,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742223_1399 (size=12151) 2024-12-08T21:16:03,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692623408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:03,684 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/f6c6051550de49c69f0398324d5e3bf8 2024-12-08T21:16:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/b322e4805c0d4dbba2b007c1cd2c3442 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/b322e4805c0d4dbba2b007c1cd2c3442 2024-12-08T21:16:03,692 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/b322e4805c0d4dbba2b007c1cd2c3442, entries=150, sequenceid=224, filesize=30.4 K 2024-12-08T21:16:03,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/e66c1a87c89f4814891223975a6e7906 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/e66c1a87c89f4814891223975a6e7906 2024-12-08T21:16:03,696 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/e66c1a87c89f4814891223975a6e7906, entries=150, sequenceid=224, filesize=11.9 K 2024-12-08T21:16:03,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/f6c6051550de49c69f0398324d5e3bf8 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/f6c6051550de49c69f0398324d5e3bf8 2024-12-08T21:16:03,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/f6c6051550de49c69f0398324d5e3bf8, entries=150, sequenceid=224, filesize=11.9 K 2024-12-08T21:16:03,701 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1295ms, sequenceid=224, compaction requested=false 2024-12-08T21:16:03,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:03,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:03,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-08T21:16:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-08T21:16:03,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-08T21:16:03,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0710 sec 2024-12-08T21:16:03,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.0770 sec 2024-12-08T21:16:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-08T21:16:03,778 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:03,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:03,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-08T21:16:03,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:03,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T21:16:03,781 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:03,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T21:16:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:03,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T21:16:03,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T21:16:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:16:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:16:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:16:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:03,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:03,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086c0a6279349c469a8bca7f29aa1e9782_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692562732/Put/seqid=0 2024-12-08T21:16:03,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742224_1400 (size=14794) 2024-12-08T21:16:04,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692623997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:04,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T21:16:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T21:16:04,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692624107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:04,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T21:16:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:04,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692624317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:04,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:04,353 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412086c0a6279349c469a8bca7f29aa1e9782_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086c0a6279349c469a8bca7f29aa1e9782_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:04,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/a5f59ceef2ea465bb4ec34aece8870b3, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:04,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/a5f59ceef2ea465bb4ec34aece8870b3 is 175, key is test_row_0/A:col10/1733692562732/Put/seqid=0 2024-12-08T21:16:04,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742225_1401 (size=39749) 2024-12-08T21:16:04,364 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/a5f59ceef2ea465bb4ec34aece8870b3 2024-12-08T21:16:04,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/11dae9efbc4249e0840a6b64c159b075 is 50, key is test_row_0/B:col10/1733692562732/Put/seqid=0 2024-12-08T21:16:04,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742226_1402 (size=12151) 2024-12-08T21:16:04,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T21:16:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:04,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T21:16:04,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T21:16:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:04,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37908 deadline: 1733692624628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:04,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:04,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T21:16:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:04,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/11dae9efbc4249e0840a6b64c159b075 2024-12-08T21:16:04,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/454253171b1644fa87c294075e60fa49 is 50, key is test_row_0/C:col10/1733692562732/Put/seqid=0 2024-12-08T21:16:04,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742227_1403 (size=12151) 2024-12-08T21:16:04,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/454253171b1644fa87c294075e60fa49 2024-12-08T21:16:04,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/a5f59ceef2ea465bb4ec34aece8870b3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/a5f59ceef2ea465bb4ec34aece8870b3 2024-12-08T21:16:04,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/a5f59ceef2ea465bb4ec34aece8870b3, entries=200, sequenceid=244, filesize=38.8 K 2024-12-08T21:16:04,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/11dae9efbc4249e0840a6b64c159b075 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/11dae9efbc4249e0840a6b64c159b075 2024-12-08T21:16:04,840 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-12-08T21:16:04,840 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:04,840 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,840 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:04,841 INFO [Thread-1527 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1527. 2024-12-08T21:16:04,841 DEBUG [Thread-1527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:04,841 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,841 DEBUG [Thread-1527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:04,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/11dae9efbc4249e0840a6b64c159b075, entries=150, sequenceid=244, filesize=11.9 K 2024-12-08T21:16:04,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/454253171b1644fa87c294075e60fa49 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/454253171b1644fa87c294075e60fa49 2024-12-08T21:16:04,844 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-12-08T21:16:04,844 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:04,844 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,845 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:04,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/454253171b1644fa87c294075e60fa49, entries=150, sequenceid=244, filesize=11.9 K 2024-12-08T21:16:04,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4fdcd6355ad16589e9afb4e1e4d173eb in 913ms, sequenceid=244, compaction requested=true 2024-12-08T21:16:04,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:04,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:04,847 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:04,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:04,847 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:04,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:04,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:04,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4fdcd6355ad16589e9afb4e1e4d173eb:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:04,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:04,848 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:04,848 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:04,848 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/B is initiating minor compaction (all files) 2024-12-08T21:16:04,848 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/A is initiating minor compaction (all files) 2024-12-08T21:16:04,848 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/A in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,848 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/B in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,848 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3938e7f0a9b94322ad397b680a80c8c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/b322e4805c0d4dbba2b007c1cd2c3442, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/a5f59ceef2ea465bb4ec34aece8870b3] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=100.1 K 2024-12-08T21:16:04,848 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/741686afbede489a8c62fe988a78531b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/e66c1a87c89f4814891223975a6e7906, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/11dae9efbc4249e0840a6b64c159b075] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=36.1 K 2024-12-08T21:16:04,849 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,849 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3938e7f0a9b94322ad397b680a80c8c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/b322e4805c0d4dbba2b007c1cd2c3442, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/a5f59ceef2ea465bb4ec34aece8870b3] 2024-12-08T21:16:04,850 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-12-08T21:16:04,850 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:04,850 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,850 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:04,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-08T21:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-08T21:16:04,851 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3938e7f0a9b94322ad397b680a80c8c7, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733692560203 2024-12-08T21:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:16:04,851 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 741686afbede489a8c62fe988a78531b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733692560203 2024-12-08T21:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:16:04,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:04,852 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-12-08T21:16:04,852 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:04,852 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b322e4805c0d4dbba2b007c1cd2c3442, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733692561465 2024-12-08T21:16:04,852 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,852 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e66c1a87c89f4814891223975a6e7906, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733692561465 2024-12-08T21:16:04,852 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:04,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:04,852 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f59ceef2ea465bb4ec34aece8870b3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733692562725 2024-12-08T21:16:04,852 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 11dae9efbc4249e0840a6b64c159b075, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733692562725 2024-12-08T21:16:04,857 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:04,861 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208b0093134a90946c891a351cfb36a10e4_4fdcd6355ad16589e9afb4e1e4d173eb store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:04,862 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208b0093134a90946c891a351cfb36a10e4_4fdcd6355ad16589e9afb4e1e4d173eb, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:04,862 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b0093134a90946c891a351cfb36a10e4_4fdcd6355ad16589e9afb4e1e4d173eb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:04,862 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:04,863 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/f6d6c78dff5d487cba6deeaa472a4cc0 is 50, key is test_row_0/B:col10/1733692562732/Put/seqid=0 2024-12-08T21:16:04,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208aabbce02b081473589a7dad608d6acf2_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692563979/Put/seqid=0 2024-12-08T21:16:04,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742228_1404 (size=12404) 2024-12-08T21:16:04,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742229_1405 (size=4469) 2024-12-08T21:16:04,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742230_1406 (size=12765) 2024-12-08T21:16:04,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:04,869 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#A#compaction#344 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:04,870 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/5970a50f5ca74feb9fcd33df15af18bc is 175, key is test_row_0/A:col10/1733692562732/Put/seqid=0 2024-12-08T21:16:04,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742231_1407 (size=31719) 2024-12-08T21:16:04,873 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208aabbce02b081473589a7dad608d6acf2_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aabbce02b081473589a7dad608d6acf2_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:04,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/ba5d046d908148ddae537964c2415c5b, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/ba5d046d908148ddae537964c2415c5b is 175, key is test_row_0/A:col10/1733692563979/Put/seqid=0 2024-12-08T21:16:04,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742232_1408 (size=31205) 2024-12-08T21:16:04,887 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/5970a50f5ca74feb9fcd33df15af18bc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/5970a50f5ca74feb9fcd33df15af18bc 2024-12-08T21:16:04,892 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/A of 4fdcd6355ad16589e9afb4e1e4d173eb into 5970a50f5ca74feb9fcd33df15af18bc(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:04,892 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:04,892 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/A, priority=13, startTime=1733692564847; duration=0sec 2024-12-08T21:16:04,892 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:04,892 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:A 2024-12-08T21:16:04,892 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:04,893 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:04,893 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 4fdcd6355ad16589e9afb4e1e4d173eb/C is initiating minor compaction (all files) 2024-12-08T21:16:04,893 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4fdcd6355ad16589e9afb4e1e4d173eb/C in TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:04,893 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b6df09aff09949168133a0301179b139, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/f6c6051550de49c69f0398324d5e3bf8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/454253171b1644fa87c294075e60fa49] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp, totalSize=36.1 K 2024-12-08T21:16:04,894 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6df09aff09949168133a0301179b139, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733692560203 2024-12-08T21:16:04,894 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c6051550de49c69f0398324d5e3bf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1733692561465 2024-12-08T21:16:04,894 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454253171b1644fa87c294075e60fa49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1733692562725 2024-12-08T21:16:04,899 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4fdcd6355ad16589e9afb4e1e4d173eb#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:04,900 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/3460f4d92cce486bbc44f065a33d17fd is 50, key is test_row_0/C:col10/1733692562732/Put/seqid=0 2024-12-08T21:16:04,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742233_1409 (size=12765) 2024-12-08T21:16:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T21:16:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:05,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. as already flushing 2024-12-08T21:16:05,139 INFO [Thread-1518 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1518. 2024-12-08T21:16:05,139 DEBUG [Thread-1518 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:05,139 DEBUG [Thread-1518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:05,139 DEBUG [Thread-1518 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:05,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:05,274 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/f6d6c78dff5d487cba6deeaa472a4cc0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/f6d6c78dff5d487cba6deeaa472a4cc0 2024-12-08T21:16:05,279 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/B of 4fdcd6355ad16589e9afb4e1e4d173eb into f6d6c78dff5d487cba6deeaa472a4cc0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:05,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:05,279 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/B, priority=13, startTime=1733692564847; duration=0sec 2024-12-08T21:16:05,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:05,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:B 2024-12-08T21:16:05,287 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/ba5d046d908148ddae537964c2415c5b 2024-12-08T21:16:05,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/309a425dcee34f87aecaa7b21b06c54d is 50, key is test_row_0/B:col10/1733692563979/Put/seqid=0 2024-12-08T21:16:05,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742234_1410 (size=12251) 2024-12-08T21:16:05,308 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/3460f4d92cce486bbc44f065a33d17fd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/3460f4d92cce486bbc44f065a33d17fd 2024-12-08T21:16:05,313 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4fdcd6355ad16589e9afb4e1e4d173eb/C of 4fdcd6355ad16589e9afb4e1e4d173eb into 3460f4d92cce486bbc44f065a33d17fd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:05,313 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:05,313 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., storeName=4fdcd6355ad16589e9afb4e1e4d173eb/C, priority=13, startTime=1733692564848; duration=0sec 2024-12-08T21:16:05,313 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:05,313 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4fdcd6355ad16589e9afb4e1e4d173eb:C 2024-12-08T21:16:05,698 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/309a425dcee34f87aecaa7b21b06c54d 2024-12-08T21:16:05,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/69935b848a30453f9064b07978e03a75 is 50, key is test_row_0/C:col10/1733692563979/Put/seqid=0 2024-12-08T21:16:05,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742235_1411 (size=12251) 2024-12-08T21:16:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T21:16:06,110 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/69935b848a30453f9064b07978e03a75 2024-12-08T21:16:06,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/ba5d046d908148ddae537964c2415c5b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/ba5d046d908148ddae537964c2415c5b 2024-12-08T21:16:06,119 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/ba5d046d908148ddae537964c2415c5b, entries=150, sequenceid=259, filesize=30.5 K 2024-12-08T21:16:06,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/309a425dcee34f87aecaa7b21b06c54d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/309a425dcee34f87aecaa7b21b06c54d 2024-12-08T21:16:06,122 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/309a425dcee34f87aecaa7b21b06c54d, entries=150, sequenceid=259, filesize=12.0 K 2024-12-08T21:16:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/69935b848a30453f9064b07978e03a75 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/69935b848a30453f9064b07978e03a75 2024-12-08T21:16:06,126 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/69935b848a30453f9064b07978e03a75, entries=150, sequenceid=259, filesize=12.0 K 2024-12-08T21:16:06,127 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1276ms, sequenceid=259, compaction requested=false 2024-12-08T21:16:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 4fdcd6355ad16589e9afb4e1e4d173eb: 2024-12-08T21:16:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-08T21:16:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-08T21:16:06,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-08T21:16:06,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3460 sec 2024-12-08T21:16:06,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3520 sec 2024-12-08T21:16:07,168 WARN [regionserver/e653e208f502:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-12-08T21:16:07,168 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/113acad0b6f842199aaaaac60e35dff3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/216203f1268d4ddaa204a505acd5e4ec, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/1781c5df08b34b1cb1a3c9cb57954fab, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/18dffd59bd8546d397b545eb3ba6d31b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/4c9c0eb55d024aecbd58520052852b1b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7480040113de4693873e1e98b33b7ab8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7a109950d0fc4f21aa4978575a15e7a7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/c25f4363c8b44b55bb82f0e3240a2a7d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3c96414b259a4187b706fe29ab0eaf65, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/8d0302f9e2054d3b93dc98fbb7b4b999, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/10c1edf86013480bbb6367af0c95f96b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/687a315eb470449893865e370dc4bad1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/efa972211f954b9f840c766565b8fd19, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/47a6d7295c674a60a765fd4953a7dee5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/0f862a46e3f14fb883f8154b9a13597a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3938e7f0a9b94322ad397b680a80c8c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/b322e4805c0d4dbba2b007c1cd2c3442, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/a5f59ceef2ea465bb4ec34aece8870b3] to archive 2024-12-08T21:16:07,169 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:16:07,171 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/113acad0b6f842199aaaaac60e35dff3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/113acad0b6f842199aaaaac60e35dff3 2024-12-08T21:16:07,172 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/216203f1268d4ddaa204a505acd5e4ec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/216203f1268d4ddaa204a505acd5e4ec 2024-12-08T21:16:07,174 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/1781c5df08b34b1cb1a3c9cb57954fab to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/1781c5df08b34b1cb1a3c9cb57954fab 2024-12-08T21:16:07,174 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/18dffd59bd8546d397b545eb3ba6d31b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/18dffd59bd8546d397b545eb3ba6d31b 2024-12-08T21:16:07,175 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/4c9c0eb55d024aecbd58520052852b1b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/4c9c0eb55d024aecbd58520052852b1b 2024-12-08T21:16:07,176 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7480040113de4693873e1e98b33b7ab8 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7480040113de4693873e1e98b33b7ab8 2024-12-08T21:16:07,177 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7a109950d0fc4f21aa4978575a15e7a7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/7a109950d0fc4f21aa4978575a15e7a7 2024-12-08T21:16:07,178 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/c25f4363c8b44b55bb82f0e3240a2a7d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/c25f4363c8b44b55bb82f0e3240a2a7d 2024-12-08T21:16:07,179 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3c96414b259a4187b706fe29ab0eaf65 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3c96414b259a4187b706fe29ab0eaf65 2024-12-08T21:16:07,180 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/8d0302f9e2054d3b93dc98fbb7b4b999 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/8d0302f9e2054d3b93dc98fbb7b4b999 2024-12-08T21:16:07,182 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/10c1edf86013480bbb6367af0c95f96b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/10c1edf86013480bbb6367af0c95f96b 2024-12-08T21:16:07,183 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/687a315eb470449893865e370dc4bad1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/687a315eb470449893865e370dc4bad1 2024-12-08T21:16:07,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/efa972211f954b9f840c766565b8fd19 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/efa972211f954b9f840c766565b8fd19 2024-12-08T21:16:07,184 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/47a6d7295c674a60a765fd4953a7dee5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/47a6d7295c674a60a765fd4953a7dee5 2024-12-08T21:16:07,185 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/0f862a46e3f14fb883f8154b9a13597a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/0f862a46e3f14fb883f8154b9a13597a 2024-12-08T21:16:07,186 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3938e7f0a9b94322ad397b680a80c8c7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/3938e7f0a9b94322ad397b680a80c8c7 2024-12-08T21:16:07,187 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/b322e4805c0d4dbba2b007c1cd2c3442 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/b322e4805c0d4dbba2b007c1cd2c3442 2024-12-08T21:16:07,188 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/a5f59ceef2ea465bb4ec34aece8870b3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/a5f59ceef2ea465bb4ec34aece8870b3 2024-12-08T21:16:07,191 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/df3c5599af0d4e8bbb6788f2ca2a1ab4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/ebad645e30fe4e65a6d065a3768a53f9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0dc69a6b96d542ada59737379df6145b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/a5e262f0c33548968a6ca55e8dbe373c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4135839c60644ed0bee2a9f0da25bd45, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dfe89aff39e44641b50a536f376282ed, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dd60d2929ff54eceb2d00793888e1196, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/021802911ac8459fa311e5031aec96fc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/071114f76a224b4cb268ccc2523b5880, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4d71b2b80d5a437ab8810861325e4ff3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/92aff8757aa34f7e92d620ea6b245c36, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/873c041d97284d05ad418720b1877a99, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0f7bf001c8514507b516426cc24ab028, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/93dc48e9dfb44ba5a25f549cd7068d77, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/741686afbede489a8c62fe988a78531b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/bec010ff3315427799db2ca9d31efea7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/e66c1a87c89f4814891223975a6e7906, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/11dae9efbc4249e0840a6b64c159b075] to archive 2024-12-08T21:16:07,196 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:16:07,197 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/df3c5599af0d4e8bbb6788f2ca2a1ab4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/df3c5599af0d4e8bbb6788f2ca2a1ab4 2024-12-08T21:16:07,198 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/ebad645e30fe4e65a6d065a3768a53f9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/ebad645e30fe4e65a6d065a3768a53f9 2024-12-08T21:16:07,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0dc69a6b96d542ada59737379df6145b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0dc69a6b96d542ada59737379df6145b 2024-12-08T21:16:07,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/a5e262f0c33548968a6ca55e8dbe373c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/a5e262f0c33548968a6ca55e8dbe373c 2024-12-08T21:16:07,201 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4135839c60644ed0bee2a9f0da25bd45 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4135839c60644ed0bee2a9f0da25bd45 2024-12-08T21:16:07,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dfe89aff39e44641b50a536f376282ed to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dfe89aff39e44641b50a536f376282ed 2024-12-08T21:16:07,203 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dd60d2929ff54eceb2d00793888e1196 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/dd60d2929ff54eceb2d00793888e1196 2024-12-08T21:16:07,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/021802911ac8459fa311e5031aec96fc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/021802911ac8459fa311e5031aec96fc 2024-12-08T21:16:07,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/071114f76a224b4cb268ccc2523b5880 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/071114f76a224b4cb268ccc2523b5880 2024-12-08T21:16:07,206 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4d71b2b80d5a437ab8810861325e4ff3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/4d71b2b80d5a437ab8810861325e4ff3 2024-12-08T21:16:07,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/92aff8757aa34f7e92d620ea6b245c36 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/92aff8757aa34f7e92d620ea6b245c36 2024-12-08T21:16:07,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/873c041d97284d05ad418720b1877a99 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/873c041d97284d05ad418720b1877a99 2024-12-08T21:16:07,208 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0f7bf001c8514507b516426cc24ab028 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/0f7bf001c8514507b516426cc24ab028 2024-12-08T21:16:07,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/93dc48e9dfb44ba5a25f549cd7068d77 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/93dc48e9dfb44ba5a25f549cd7068d77 2024-12-08T21:16:07,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/741686afbede489a8c62fe988a78531b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/741686afbede489a8c62fe988a78531b 2024-12-08T21:16:07,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/bec010ff3315427799db2ca9d31efea7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/bec010ff3315427799db2ca9d31efea7 2024-12-08T21:16:07,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/e66c1a87c89f4814891223975a6e7906 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/e66c1a87c89f4814891223975a6e7906 2024-12-08T21:16:07,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/11dae9efbc4249e0840a6b64c159b075 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/11dae9efbc4249e0840a6b64c159b075 2024-12-08T21:16:07,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/27d0f02f96b24a9a8eb2c3c36738f592, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/371586d6117b4878a4064806a82ddf17, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/7580d391bb66491399c7ea0601211f18, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/6551ea8063ed4d998285380872c99d09, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a061880279540219706b996c0abfbd2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b55c7688fb214a938fe9ac8e287bab3c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c55620accbd5486a916c2e9442d37b95, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c1c2c50c1a4244f080a1b2262be6cf14, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/8ac8c206b9d6496d95e8e95d5bd61d1d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/eac733badfa34dc8b8888e4fa191ce79, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a38410649484e88844b0fc081ee6212, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/1c028ef03fab4330988558c8247dcaec, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/9c01d19ea339446e9689c6934ab77b56, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/82e46bfb829a4c21b23076c15c606902, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b6df09aff09949168133a0301179b139, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/251eb7ad9855493d8b7f678df67e5176, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/f6c6051550de49c69f0398324d5e3bf8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/454253171b1644fa87c294075e60fa49] to archive 2024-12-08T21:16:07,215 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:16:07,216 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/27d0f02f96b24a9a8eb2c3c36738f592 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/27d0f02f96b24a9a8eb2c3c36738f592 2024-12-08T21:16:07,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/371586d6117b4878a4064806a82ddf17 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/371586d6117b4878a4064806a82ddf17 2024-12-08T21:16:07,217 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/7580d391bb66491399c7ea0601211f18 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/7580d391bb66491399c7ea0601211f18 2024-12-08T21:16:07,218 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/6551ea8063ed4d998285380872c99d09 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/6551ea8063ed4d998285380872c99d09 2024-12-08T21:16:07,219 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a061880279540219706b996c0abfbd2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a061880279540219706b996c0abfbd2 2024-12-08T21:16:07,220 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b55c7688fb214a938fe9ac8e287bab3c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b55c7688fb214a938fe9ac8e287bab3c 2024-12-08T21:16:07,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c55620accbd5486a916c2e9442d37b95 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c55620accbd5486a916c2e9442d37b95 2024-12-08T21:16:07,221 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c1c2c50c1a4244f080a1b2262be6cf14 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/c1c2c50c1a4244f080a1b2262be6cf14 2024-12-08T21:16:07,222 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/8ac8c206b9d6496d95e8e95d5bd61d1d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/8ac8c206b9d6496d95e8e95d5bd61d1d 2024-12-08T21:16:07,223 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/eac733badfa34dc8b8888e4fa191ce79 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/eac733badfa34dc8b8888e4fa191ce79 2024-12-08T21:16:07,224 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a38410649484e88844b0fc081ee6212 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/0a38410649484e88844b0fc081ee6212 2024-12-08T21:16:07,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/1c028ef03fab4330988558c8247dcaec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/1c028ef03fab4330988558c8247dcaec 2024-12-08T21:16:07,225 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/9c01d19ea339446e9689c6934ab77b56 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/9c01d19ea339446e9689c6934ab77b56 2024-12-08T21:16:07,226 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/82e46bfb829a4c21b23076c15c606902 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/82e46bfb829a4c21b23076c15c606902 2024-12-08T21:16:07,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b6df09aff09949168133a0301179b139 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/b6df09aff09949168133a0301179b139 2024-12-08T21:16:07,227 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/251eb7ad9855493d8b7f678df67e5176 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/251eb7ad9855493d8b7f678df67e5176 2024-12-08T21:16:07,228 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/f6c6051550de49c69f0398324d5e3bf8 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/f6c6051550de49c69f0398324d5e3bf8 2024-12-08T21:16:07,229 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/e653e208f502:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/454253171b1644fa87c294075e60fa49 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/454253171b1644fa87c294075e60fa49 2024-12-08T21:16:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-08T21:16:07,948 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:12,069 INFO [Thread-1522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1522. 2024-12-08T21:16:12,069 DEBUG [Thread-1522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:12,069 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:12,069 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:12,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:12,140 INFO [Thread-1516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1516. 2024-12-08T21:16:12,140 DEBUG [Thread-1516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:12,140 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:12,140 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:12,140 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:13,243 INFO [Thread-1520 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1520. 2024-12-08T21:16:13,243 DEBUG [Thread-1520 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:13,243 DEBUG [Thread-1520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:13,244 DEBUG [Thread-1520 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:13,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:13,293 INFO [Thread-1524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1524. 2024-12-08T21:16:13,293 DEBUG [Thread-1524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:13,293 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:13,293 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:13,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1640 2024-12-08T21:16:13,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4916 rows 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1639 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4913 rows 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1646 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4934 rows 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1649 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4943 rows 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1648 2024-12-08T21:16:13,295 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4944 rows 2024-12-08T21:16:13,295 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:16:13,295 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:16:13,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:13,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:13,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:13,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T21:16:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T21:16:13,302 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692573302"}]},"ts":"1733692573302"} 2024-12-08T21:16:13,303 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T21:16:13,303 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T21:16:13,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:16:13,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, UNASSIGN}] 2024-12-08T21:16:13,307 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, UNASSIGN 2024-12-08T21:16:13,308 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:13,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, UNASSIGN because future has completed 2024-12-08T21:16:13,310 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:16:13,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:16:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T21:16:13,463 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:13,463 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:16:13,463 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 4fdcd6355ad16589e9afb4e1e4d173eb, disabling compactions & flushes 2024-12-08T21:16:13,464 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:13,464 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:13,464 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. after waiting 0 ms 2024-12-08T21:16:13,464 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:13,464 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 4fdcd6355ad16589e9afb4e1e4d173eb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T21:16:13,465 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=A 2024-12-08T21:16:13,465 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:13,465 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=B 2024-12-08T21:16:13,465 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:13,465 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4fdcd6355ad16589e9afb4e1e4d173eb, store=C 2024-12-08T21:16:13,465 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:13,473 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c31e33cdc9cb45f9bbf52a0409bdf627_4fdcd6355ad16589e9afb4e1e4d173eb is 50, key is test_row_0/A:col10/1733692572139/Put/seqid=0 2024-12-08T21:16:13,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742236_1412 (size=12454) 2024-12-08T21:16:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T21:16:13,878 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:13,883 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c31e33cdc9cb45f9bbf52a0409bdf627_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c31e33cdc9cb45f9bbf52a0409bdf627_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:13,883 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/120c2c30aee54001b6321c9944ef3024, store: [table=TestAcidGuarantees family=A region=4fdcd6355ad16589e9afb4e1e4d173eb] 2024-12-08T21:16:13,884 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/120c2c30aee54001b6321c9944ef3024 is 175, key is test_row_0/A:col10/1733692572139/Put/seqid=0 2024-12-08T21:16:13,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742237_1413 (size=31255) 2024-12-08T21:16:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T21:16:14,288 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/120c2c30aee54001b6321c9944ef3024 2024-12-08T21:16:14,294 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/353b8b7302234b98a4fbcd6e1ab27de1 is 50, key is test_row_0/B:col10/1733692572139/Put/seqid=0 2024-12-08T21:16:14,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742238_1414 (size=12301) 2024-12-08T21:16:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T21:16:14,700 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/353b8b7302234b98a4fbcd6e1ab27de1 2024-12-08T21:16:14,706 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/67252f23a578467ca6dd0151165ad7a1 is 50, key is test_row_0/C:col10/1733692572139/Put/seqid=0 2024-12-08T21:16:14,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742239_1415 (size=12301) 2024-12-08T21:16:15,109 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/67252f23a578467ca6dd0151165ad7a1 2024-12-08T21:16:15,113 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/A/120c2c30aee54001b6321c9944ef3024 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/120c2c30aee54001b6321c9944ef3024 2024-12-08T21:16:15,117 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/120c2c30aee54001b6321c9944ef3024, entries=150, sequenceid=270, filesize=30.5 K 2024-12-08T21:16:15,118 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/B/353b8b7302234b98a4fbcd6e1ab27de1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/353b8b7302234b98a4fbcd6e1ab27de1 2024-12-08T21:16:15,123 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/353b8b7302234b98a4fbcd6e1ab27de1, entries=150, sequenceid=270, filesize=12.0 K 2024-12-08T21:16:15,124 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/.tmp/C/67252f23a578467ca6dd0151165ad7a1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/67252f23a578467ca6dd0151165ad7a1 2024-12-08T21:16:15,128 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/67252f23a578467ca6dd0151165ad7a1, entries=150, sequenceid=270, filesize=12.0 K 2024-12-08T21:16:15,129 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1665ms, sequenceid=270, compaction requested=true 2024-12-08T21:16:15,132 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/recovered.edits/273.seqid, newMaxSeqId=273, maxSeqId=4 2024-12-08T21:16:15,133 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. 2024-12-08T21:16:15,133 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 4fdcd6355ad16589e9afb4e1e4d173eb: Waiting for close lock at 1733692573463Running coprocessor pre-close hooks at 1733692573463Disabling compacts and flushes for region at 1733692573463Disabling writes for close at 1733692573464 (+1 ms)Obtaining lock to block concurrent updates at 1733692573464Preparing flush snapshotting stores in 4fdcd6355ad16589e9afb4e1e4d173eb at 1733692573464Finished memstore snapshotting TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733692573465 (+1 ms)Flushing stores of TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb. at 1733692573467 (+2 ms)Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/A: creating writer at 1733692573467Flushing mob file 4fdcd6355ad16589e9afb4e1e4d173eb/A: appending metadata at 1733692573473 (+6 ms)Flushing mob file 4fdcd6355ad16589e9afb4e1e4d173eb/A: closing flushed file at 1733692573473Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/A: appending metadata at 1733692573883 (+410 ms)Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/A: closing flushed file at 1733692573884 (+1 ms)Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/B: creating writer at 1733692574291 (+407 ms)Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/B: appending metadata at 1733692574293 (+2 ms)Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/B: closing flushed file at 1733692574293Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/C: creating writer at 1733692574703 (+410 ms)Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/C: appending metadata at 1733692574705 (+2 ms)Flushing 4fdcd6355ad16589e9afb4e1e4d173eb/C: closing flushed file at 1733692574705Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@456fc7b2: reopening flushed file at 1733692575113 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29820ca3: reopening flushed file at 1733692575117 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a0cb3c2: reopening flushed file at 1733692575123 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4fdcd6355ad16589e9afb4e1e4d173eb in 1665ms, sequenceid=270, compaction requested=true at 1733692575129 (+6 ms)Writing region close event to WAL at 1733692575130 (+1 ms)Running coprocessor post-close hooks at 1733692575133 (+3 ms)Closed at 1733692575133 2024-12-08T21:16:15,134 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,135 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=4fdcd6355ad16589e9afb4e1e4d173eb, regionState=CLOSED 2024-12-08T21:16:15,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:16:15,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-08T21:16:15,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 4fdcd6355ad16589e9afb4e1e4d173eb, server=e653e208f502,38457,1733692444075 in 1.8270 sec 2024-12-08T21:16:15,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-08T21:16:15,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4fdcd6355ad16589e9afb4e1e4d173eb, UNASSIGN in 1.8320 sec 2024-12-08T21:16:15,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-08T21:16:15,142 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8360 sec 2024-12-08T21:16:15,143 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692575143"}]},"ts":"1733692575143"} 2024-12-08T21:16:15,144 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T21:16:15,144 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T21:16:15,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8460 sec 2024-12-08T21:16:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-08T21:16:15,448 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:15,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T21:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:15,450 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T21:16:15,451 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:15,453 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,455 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/recovered.edits] 2024-12-08T21:16:15,457 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/120c2c30aee54001b6321c9944ef3024 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/120c2c30aee54001b6321c9944ef3024 2024-12-08T21:16:15,458 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/5970a50f5ca74feb9fcd33df15af18bc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/5970a50f5ca74feb9fcd33df15af18bc 2024-12-08T21:16:15,459 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/ba5d046d908148ddae537964c2415c5b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/A/ba5d046d908148ddae537964c2415c5b 2024-12-08T21:16:15,461 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/309a425dcee34f87aecaa7b21b06c54d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/309a425dcee34f87aecaa7b21b06c54d 2024-12-08T21:16:15,461 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/353b8b7302234b98a4fbcd6e1ab27de1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/353b8b7302234b98a4fbcd6e1ab27de1 2024-12-08T21:16:15,462 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/f6d6c78dff5d487cba6deeaa472a4cc0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/B/f6d6c78dff5d487cba6deeaa472a4cc0 2024-12-08T21:16:15,464 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/3460f4d92cce486bbc44f065a33d17fd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/3460f4d92cce486bbc44f065a33d17fd 2024-12-08T21:16:15,465 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/67252f23a578467ca6dd0151165ad7a1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/67252f23a578467ca6dd0151165ad7a1 2024-12-08T21:16:15,466 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/69935b848a30453f9064b07978e03a75 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/C/69935b848a30453f9064b07978e03a75 2024-12-08T21:16:15,468 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/recovered.edits/273.seqid to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb/recovered.edits/273.seqid 2024-12-08T21:16:15,469 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,469 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T21:16:15,469 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T21:16:15,470 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T21:16:15,472 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208042d5687463146e5bc55bbf9a6c979f4_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208042d5687463146e5bc55bbf9a6c979f4_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,473 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081959c27137a149cabf681549b9e280f2_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081959c27137a149cabf681549b9e280f2_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,474 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120825477c44b87b4ebbbac6bbcc4ccada02_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120825477c44b87b4ebbbac6bbcc4ccada02_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,475 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208277b9fc973be4142ba50dc1d0d812b43_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208277b9fc973be4142ba50dc1d0d812b43_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,476 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120828118d2d380242aaacfa2682ebb7a695_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120828118d2d380242aaacfa2682ebb7a695_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,478 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120844f3de91a53e496bb1d12d39f1318af1_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120844f3de91a53e496bb1d12d39f1318af1_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,479 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120857a0b8e195e24160b30c61871f104012_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120857a0b8e195e24160b30c61871f104012_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,480 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120858f062026b2b49b294b3ec15122a6abd_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120858f062026b2b49b294b3ec15122a6abd_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,481 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085a4ae75e090b44caa85f7fb3a5dd3f6f_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412085a4ae75e090b44caa85f7fb3a5dd3f6f_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,482 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086beca0999d4e4c70836d8e794cf0696c_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086beca0999d4e4c70836d8e794cf0696c_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,483 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086c0a6279349c469a8bca7f29aa1e9782_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412086c0a6279349c469a8bca7f29aa1e9782_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,484 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089071015deb5d44a18b4c8338754dcd11_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412089071015deb5d44a18b4c8338754dcd11_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,486 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aabbce02b081473589a7dad608d6acf2_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208aabbce02b081473589a7dad608d6acf2_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,486 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bcd5f9f2cc0a4d969e3c42a65c552943_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bcd5f9f2cc0a4d969e3c42a65c552943_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,487 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c31e33cdc9cb45f9bbf52a0409bdf627_4fdcd6355ad16589e9afb4e1e4d173eb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c31e33cdc9cb45f9bbf52a0409bdf627_4fdcd6355ad16589e9afb4e1e4d173eb 2024-12-08T21:16:15,488 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T21:16:15,490 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:15,492 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T21:16:15,494 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T21:16:15,495 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:15,496 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T21:16:15,496 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733692575496"}]},"ts":"9223372036854775807"} 2024-12-08T21:16:15,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T21:16:15,498 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4fdcd6355ad16589e9afb4e1e4d173eb, NAME => 'TestAcidGuarantees,,1733692541708.4fdcd6355ad16589e9afb4e1e4d173eb.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T21:16:15,498 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T21:16:15,498 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733692575498"}]},"ts":"9223372036854775807"} 2024-12-08T21:16:15,500 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T21:16:15,501 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:15,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-12-08T21:16:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-08T21:16:15,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T21:16:15,558 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:15,567 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=443 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=472 (was 481), ProcessCount=11 (was 11), AvailableMemoryMB=8946 (was 7860) - AvailableMemoryMB LEAK? - 2024-12-08T21:16:15,577 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=472, ProcessCount=11, AvailableMemoryMB=8943 2024-12-08T21:16:15,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:16:15,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:16:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:15,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:16:15,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-12-08T21:16:15,581 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T21:16:15,582 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:16:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742240_1416 (size=1061) 2024-12-08T21:16:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T21:16:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T21:16:15,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:16:15,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742241_1417 (size=53) 2024-12-08T21:16:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T21:16:16,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:16:16,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c482ca018e010a4a9e75b04b5a867252, disabling compactions & flushes 2024-12-08T21:16:16,399 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. after waiting 0 ms 2024-12-08T21:16:16,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,399 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,399 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c482ca018e010a4a9e75b04b5a867252: Waiting for close lock at 1733692576399Disabling compacts and flushes for region at 1733692576399Disabling writes for close at 1733692576399Writing region close event to WAL at 1733692576399Closed at 1733692576399 2024-12-08T21:16:16,400 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:16:16,401 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733692576401"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733692576401"}]},"ts":"1733692576401"} 2024-12-08T21:16:16,402 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:16:16,404 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:16:16,404 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692576404"}]},"ts":"1733692576404"} 2024-12-08T21:16:16,405 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T21:16:16,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, ASSIGN}] 2024-12-08T21:16:16,407 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, ASSIGN 2024-12-08T21:16:16,408 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, ASSIGN; state=OFFLINE, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=false 2024-12-08T21:16:16,559 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=c482ca018e010a4a9e75b04b5a867252, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:16,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, ASSIGN because future has completed 2024-12-08T21:16:16,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:16:16,716 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,716 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:16:16,717 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,717 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:16:16,717 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,717 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T21:16:16,718 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,720 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:16,720 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c482ca018e010a4a9e75b04b5a867252 columnFamilyName A 2024-12-08T21:16:16,720 DEBUG [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:16,720 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.HStore(327): Store=c482ca018e010a4a9e75b04b5a867252/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:16,721 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,722 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:16,722 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c482ca018e010a4a9e75b04b5a867252 columnFamilyName B 2024-12-08T21:16:16,722 DEBUG [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:16,722 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.HStore(327): Store=c482ca018e010a4a9e75b04b5a867252/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:16,722 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,723 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:16,724 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c482ca018e010a4a9e75b04b5a867252 columnFamilyName C 2024-12-08T21:16:16,724 DEBUG [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:16,724 INFO [StoreOpener-c482ca018e010a4a9e75b04b5a867252-1 {}] regionserver.HStore(327): Store=c482ca018e010a4a9e75b04b5a867252/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:16,724 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,724 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,725 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,725 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,726 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,727 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,727 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:16:16,728 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,731 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:16:16,732 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened c482ca018e010a4a9e75b04b5a867252; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72693713, jitterRate=0.08322073519229889}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:16:16,732 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:16,732 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for c482ca018e010a4a9e75b04b5a867252: Running coprocessor pre-open hook at 1733692576717Writing region info on filesystem at 1733692576717Initializing all the Stores at 1733692576718 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692576718Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692576718Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692576718Cleaning up temporary data from old regions at 1733692576727 (+9 ms)Running coprocessor post-open hooks at 1733692576732 (+5 ms)Region opened successfully at 1733692576732 2024-12-08T21:16:16,733 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., pid=116, masterSystemTime=1733692576713 2024-12-08T21:16:16,735 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,735 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:16,736 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=c482ca018e010a4a9e75b04b5a867252, regionState=OPEN, openSeqNum=2, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:16,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:16:16,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-08T21:16:16,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 in 177 msec 2024-12-08T21:16:16,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-08T21:16:16,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, ASSIGN in 334 msec 2024-12-08T21:16:16,743 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:16:16,743 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692576743"}]},"ts":"1733692576743"} 2024-12-08T21:16:16,744 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T21:16:16,745 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:16:16,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-12-08T21:16:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-08T21:16:17,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:17,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,732 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3610338e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,734 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd5e51e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:16:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e653e208f502,45981,1733692443391 2024-12-08T21:16:17,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3296eb6d 2024-12-08T21:16:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:16:17,743 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:16:17,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e31baff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,749 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f36488, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,751 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2e302c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,753 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff8fe4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,756 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0548c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,757 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de468ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,759 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6afc28fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2608f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,761 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5453be20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,762 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74106992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,764 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76ae4ce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,765 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e001fae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,767 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28682cc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,768 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d9e018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,769 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3943dae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,770 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a92873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,772 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6069e036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,773 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28b65bda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d70caa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:17,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:17,774 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa05bba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:17,775 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f7a1ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:17,776 DEBUG [Thread-1889 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,776 DEBUG [Thread-1902 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,776 DEBUG [Thread-1897 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,776 DEBUG [Thread-1906 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:17,776 DEBUG [Thread-1893 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,777 DEBUG [Thread-1904 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,777 DEBUG [Thread-1895 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,777 DEBUG [Thread-1908 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-08T21:16:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,778 DEBUG [Thread-1895 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,778 DEBUG [Thread-1895 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,779 DEBUG [Thread-1900 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T21:16:17,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,780 DEBUG [Thread-1891 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:17,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:17,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,783 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:17,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:17,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2] 2024-12-08T21:16:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:17,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T21:16:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:17,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:17,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/9c165f30f9414fd18a0e123444534b49 is 50, key is test_row_0/A:col10/1733692577789/Put/seqid=0 2024-12-08T21:16:17,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742242_1418 (size=12001) 2024-12-08T21:16:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692637836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692637837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692637838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692637840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692637841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T21:16:17,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T21:16:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:17,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:17,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692637947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692637948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692637948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692637948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:17,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692637958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:17,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T21:16:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:18,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T21:16:18,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692638158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692638158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692638158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692638158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692638167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/9c165f30f9414fd18a0e123444534b49 2024-12-08T21:16:18,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/ae69b99a92ac4731a1591bcd38c15b4e is 50, key is test_row_0/B:col10/1733692577789/Put/seqid=0 2024-12-08T21:16:18,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742243_1419 (size=12001) 2024-12-08T21:16:18,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T21:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T21:16:18,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:18,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T21:16:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692638468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692638468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692638468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692638468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692638477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T21:16:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:18,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/ae69b99a92ac4731a1591bcd38c15b4e 2024-12-08T21:16:18,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/8316f537b205443fa9309d145d96b6e9 is 50, key is test_row_0/C:col10/1733692577789/Put/seqid=0 2024-12-08T21:16:18,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742244_1420 (size=12001) 2024-12-08T21:16:18,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/8316f537b205443fa9309d145d96b6e9 2024-12-08T21:16:18,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/9c165f30f9414fd18a0e123444534b49 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9c165f30f9414fd18a0e123444534b49 2024-12-08T21:16:18,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9c165f30f9414fd18a0e123444534b49, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T21:16:18,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/ae69b99a92ac4731a1591bcd38c15b4e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/ae69b99a92ac4731a1591bcd38c15b4e 2024-12-08T21:16:18,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/ae69b99a92ac4731a1591bcd38c15b4e, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T21:16:18,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/8316f537b205443fa9309d145d96b6e9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8316f537b205443fa9309d145d96b6e9 2024-12-08T21:16:18,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8316f537b205443fa9309d145d96b6e9, entries=150, sequenceid=13, filesize=11.7 K 2024-12-08T21:16:18,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c482ca018e010a4a9e75b04b5a867252 in 877ms, sequenceid=13, compaction requested=false 2024-12-08T21:16:18,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:18,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-08T21:16:18,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:18,701 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:16:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:18,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/35cfb1beab4e40ab89f1d98794adea46 is 50, key is test_row_0/A:col10/1733692577839/Put/seqid=0 2024-12-08T21:16:18,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742245_1421 (size=12001) 2024-12-08T21:16:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T21:16:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:18,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692638988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692638988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692638991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692638992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692638992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:18,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692639098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692639098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,108 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/35cfb1beab4e40ab89f1d98794adea46 2024-12-08T21:16:19,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692639108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692639108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/cec4c1099eb84586a5d3936115f25d05 is 50, key is test_row_0/B:col10/1733692577839/Put/seqid=0 2024-12-08T21:16:19,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742246_1422 (size=12001) 2024-12-08T21:16:19,121 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/cec4c1099eb84586a5d3936115f25d05 2024-12-08T21:16:19,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f44d0b61c59740c785fbd511f4cf495a is 50, key is test_row_0/C:col10/1733692577839/Put/seqid=0 2024-12-08T21:16:19,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742247_1423 (size=12001) 2024-12-08T21:16:19,135 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f44d0b61c59740c785fbd511f4cf495a 2024-12-08T21:16:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/35cfb1beab4e40ab89f1d98794adea46 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/35cfb1beab4e40ab89f1d98794adea46 2024-12-08T21:16:19,142 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/35cfb1beab4e40ab89f1d98794adea46, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T21:16:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/cec4c1099eb84586a5d3936115f25d05 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/cec4c1099eb84586a5d3936115f25d05 2024-12-08T21:16:19,146 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/cec4c1099eb84586a5d3936115f25d05, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T21:16:19,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f44d0b61c59740c785fbd511f4cf495a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f44d0b61c59740c785fbd511f4cf495a 2024-12-08T21:16:19,152 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f44d0b61c59740c785fbd511f4cf495a, entries=150, sequenceid=33, filesize=11.7 K 2024-12-08T21:16:19,152 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c482ca018e010a4a9e75b04b5a867252 in 451ms, sequenceid=33, compaction requested=false 2024-12-08T21:16:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-08T21:16:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-08T21:16:19,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-08T21:16:19,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3710 sec 2024-12-08T21:16:19,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.3800 sec 2024-12-08T21:16:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:19,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:16:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:19,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:19,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/ac073ff542344d739a78d5f535b46179 is 50, key is test_row_0/A:col10/1733692579311/Put/seqid=0 2024-12-08T21:16:19,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742248_1424 (size=16681) 2024-12-08T21:16:19,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692639341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692639342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692639342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692639342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692639457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692639458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692639458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692639458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692639667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692639667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692639667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692639678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/ac073ff542344d739a78d5f535b46179 2024-12-08T21:16:19,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/1777fd4e110446d9a209f005d084d121 is 50, key is test_row_0/B:col10/1733692579311/Put/seqid=0 2024-12-08T21:16:19,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742249_1425 (size=12001) 2024-12-08T21:16:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-08T21:16:19,928 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:19,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-08T21:16:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T21:16:19,930 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:19,930 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:19,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:19,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692639978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692639978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692639979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:19,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692639988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:19,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692640008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:20,010 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:20,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T21:16:20,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T21:16:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:20,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/1777fd4e110446d9a209f005d084d121 2024-12-08T21:16:20,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/5a061befdf2047b8883e2f021598a9f5 is 50, key is test_row_0/C:col10/1733692579311/Put/seqid=0 2024-12-08T21:16:20,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742250_1426 (size=12001) 2024-12-08T21:16:20,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T21:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T21:16:20,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T21:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692640487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:20,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692640498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692640498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692640498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:20,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T21:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:20,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/5a061befdf2047b8883e2f021598a9f5 2024-12-08T21:16:20,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/ac073ff542344d739a78d5f535b46179 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ac073ff542344d739a78d5f535b46179 2024-12-08T21:16:20,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ac073ff542344d739a78d5f535b46179, entries=250, sequenceid=47, filesize=16.3 K 2024-12-08T21:16:20,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/1777fd4e110446d9a209f005d084d121 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/1777fd4e110446d9a209f005d084d121 2024-12-08T21:16:20,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/1777fd4e110446d9a209f005d084d121, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T21:16:20,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/5a061befdf2047b8883e2f021598a9f5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/5a061befdf2047b8883e2f021598a9f5 2024-12-08T21:16:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T21:16:20,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/5a061befdf2047b8883e2f021598a9f5, entries=150, sequenceid=47, filesize=11.7 K 2024-12-08T21:16:20,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c482ca018e010a4a9e75b04b5a867252 in 1247ms, sequenceid=47, compaction requested=true 2024-12-08T21:16:20,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:20,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:20,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:20,560 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:20,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:20,560 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:20,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:20,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:20,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:20,561 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:20,561 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/B is initiating minor compaction (all files) 2024-12-08T21:16:20,561 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/B in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,561 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/ae69b99a92ac4731a1591bcd38c15b4e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/cec4c1099eb84586a5d3936115f25d05, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/1777fd4e110446d9a209f005d084d121] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=35.2 K 2024-12-08T21:16:20,561 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:20,561 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/A is initiating minor compaction (all files) 2024-12-08T21:16:20,561 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/A in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,561 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9c165f30f9414fd18a0e123444534b49, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/35cfb1beab4e40ab89f1d98794adea46, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ac073ff542344d739a78d5f535b46179] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=39.7 K 2024-12-08T21:16:20,563 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ae69b99a92ac4731a1591bcd38c15b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733692577787 2024-12-08T21:16:20,563 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c165f30f9414fd18a0e123444534b49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733692577787 2024-12-08T21:16:20,563 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting cec4c1099eb84586a5d3936115f25d05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733692577837 2024-12-08T21:16:20,563 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35cfb1beab4e40ab89f1d98794adea46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733692577837 2024-12-08T21:16:20,563 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac073ff542344d739a78d5f535b46179, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692578982 2024-12-08T21:16:20,564 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 1777fd4e110446d9a209f005d084d121, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692578990 2024-12-08T21:16:20,570 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#A#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:20,571 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/ffbfd992dbba4d66a25352074a9ad645 is 50, key is test_row_0/A:col10/1733692579311/Put/seqid=0 2024-12-08T21:16:20,573 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:20,573 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/b248195fff074e329e5b90380b6881b0 is 50, key is test_row_0/B:col10/1733692579311/Put/seqid=0 2024-12-08T21:16:20,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742252_1428 (size=12104) 2024-12-08T21:16:20,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742251_1427 (size=12104) 2024-12-08T21:16:20,598 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/ffbfd992dbba4d66a25352074a9ad645 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ffbfd992dbba4d66a25352074a9ad645 2024-12-08T21:16:20,601 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/A of c482ca018e010a4a9e75b04b5a867252 into ffbfd992dbba4d66a25352074a9ad645(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:20,601 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:20,601 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/A, priority=13, startTime=1733692580560; duration=0sec 2024-12-08T21:16:20,602 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:20,602 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:20,602 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:20,608 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:20,609 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/C is initiating minor compaction (all files) 2024-12-08T21:16:20,609 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/C in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,609 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8316f537b205443fa9309d145d96b6e9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f44d0b61c59740c785fbd511f4cf495a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/5a061befdf2047b8883e2f021598a9f5] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=35.2 K 2024-12-08T21:16:20,609 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8316f537b205443fa9309d145d96b6e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733692577787 2024-12-08T21:16:20,609 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44d0b61c59740c785fbd511f4cf495a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733692577837 2024-12-08T21:16:20,610 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a061befdf2047b8883e2f021598a9f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692578990 2024-12-08T21:16:20,621 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#C#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:20,621 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/23bea873a4da4c88b077c3efe8346f82 is 50, key is test_row_0/C:col10/1733692579311/Put/seqid=0 2024-12-08T21:16:20,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742253_1429 (size=12104) 2024-12-08T21:16:20,652 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:16:20,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-08T21:16:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:20,694 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T21:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:20,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:20,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/02feb66403f44babbdfa40549c79c9fd is 50, key is test_row_0/A:col10/1733692579341/Put/seqid=0 2024-12-08T21:16:20,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742254_1430 (size=12001) 2024-12-08T21:16:20,744 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/02feb66403f44babbdfa40549c79c9fd 2024-12-08T21:16:20,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/169e84455c774f10ba712229f49c12e0 is 50, key is test_row_0/B:col10/1733692579341/Put/seqid=0 2024-12-08T21:16:20,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742255_1431 (size=12001) 2024-12-08T21:16:20,990 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/b248195fff074e329e5b90380b6881b0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b248195fff074e329e5b90380b6881b0 2024-12-08T21:16:20,996 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/B of c482ca018e010a4a9e75b04b5a867252 into b248195fff074e329e5b90380b6881b0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:20,996 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:20,996 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/B, priority=13, startTime=1733692580560; duration=0sec 2024-12-08T21:16:20,997 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:20,997 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:21,063 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/23bea873a4da4c88b077c3efe8346f82 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/23bea873a4da4c88b077c3efe8346f82 2024-12-08T21:16:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T21:16:21,069 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/C of c482ca018e010a4a9e75b04b5a867252 into 23bea873a4da4c88b077c3efe8346f82(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:21,069 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:21,069 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/C, priority=13, startTime=1733692580560; duration=0sec 2024-12-08T21:16:21,069 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:21,069 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:21,216 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/169e84455c774f10ba712229f49c12e0 2024-12-08T21:16:21,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/cc2d6d5142de4475a91ce592ec6f58c7 is 50, key is test_row_0/C:col10/1733692579341/Put/seqid=0 2024-12-08T21:16:21,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742256_1432 (size=12001) 2024-12-08T21:16:21,291 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/cc2d6d5142de4475a91ce592ec6f58c7 2024-12-08T21:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/02feb66403f44babbdfa40549c79c9fd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/02feb66403f44babbdfa40549c79c9fd 2024-12-08T21:16:21,312 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/02feb66403f44babbdfa40549c79c9fd, entries=150, sequenceid=66, filesize=11.7 K 2024-12-08T21:16:21,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/169e84455c774f10ba712229f49c12e0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/169e84455c774f10ba712229f49c12e0 2024-12-08T21:16:21,323 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/169e84455c774f10ba712229f49c12e0, entries=150, sequenceid=66, filesize=11.7 K 2024-12-08T21:16:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/cc2d6d5142de4475a91ce592ec6f58c7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/cc2d6d5142de4475a91ce592ec6f58c7 2024-12-08T21:16:21,335 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/cc2d6d5142de4475a91ce592ec6f58c7, entries=150, sequenceid=66, filesize=11.7 K 2024-12-08T21:16:21,336 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for c482ca018e010a4a9e75b04b5a867252 in 643ms, sequenceid=66, compaction requested=false 2024-12-08T21:16:21,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:21,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:21,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-08T21:16:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-08T21:16:21,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-08T21:16:21,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-12-08T21:16:21,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4160 sec 2024-12-08T21:16:21,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:16:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:21,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/c31e70a624f7429bad1b852685cd0c66 is 50, key is test_row_0/A:col10/1733692581521/Put/seqid=0 2024-12-08T21:16:21,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742257_1433 (size=16681) 2024-12-08T21:16:21,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/c31e70a624f7429bad1b852685cd0c66 2024-12-08T21:16:21,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/e87c8f7a060a4208a961c757bfe3ee5f is 50, key is test_row_0/B:col10/1733692581521/Put/seqid=0 2024-12-08T21:16:21,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692581547, max timestamp=1733692581586Num uniques 859; 2024-12-08T21:16:21,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:21,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692581547, max timestamp=1733692581588Num uniques 859; 2024-12-08T21:16:21,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692581547, max timestamp=1733692581588Num uniques 859; 2024-12-08T21:16:21,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:21,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:21,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742258_1434 (size=12001) 2024-12-08T21:16:21,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/e87c8f7a060a4208a961c757bfe3ee5f 2024-12-08T21:16:21,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/054b50d6de904c4ba82fbc3d1178bff7 is 50, key is test_row_0/C:col10/1733692581521/Put/seqid=0 2024-12-08T21:16:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692641651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692641651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692641652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692641664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742259_1435 (size=12001) 2024-12-08T21:16:21,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/054b50d6de904c4ba82fbc3d1178bff7 2024-12-08T21:16:21,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/c31e70a624f7429bad1b852685cd0c66 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c31e70a624f7429bad1b852685cd0c66 2024-12-08T21:16:21,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c31e70a624f7429bad1b852685cd0c66, entries=250, sequenceid=78, filesize=16.3 K 2024-12-08T21:16:21,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/e87c8f7a060a4208a961c757bfe3ee5f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e87c8f7a060a4208a961c757bfe3ee5f 2024-12-08T21:16:21,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e87c8f7a060a4208a961c757bfe3ee5f, entries=150, sequenceid=78, filesize=11.7 K 2024-12-08T21:16:21,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/054b50d6de904c4ba82fbc3d1178bff7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/054b50d6de904c4ba82fbc3d1178bff7 2024-12-08T21:16:21,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/054b50d6de904c4ba82fbc3d1178bff7, entries=150, sequenceid=78, filesize=11.7 K 2024-12-08T21:16:21,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c482ca018e010a4a9e75b04b5a867252 in 188ms, sequenceid=78, compaction requested=true 2024-12-08T21:16:21,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:21,710 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:21,711 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:21,712 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/A is initiating minor compaction (all files) 2024-12-08T21:16:21,712 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/A in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:21,712 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ffbfd992dbba4d66a25352074a9ad645, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/02feb66403f44babbdfa40549c79c9fd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c31e70a624f7429bad1b852685cd0c66] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=39.8 K 2024-12-08T21:16:21,712 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffbfd992dbba4d66a25352074a9ad645, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692578990 2024-12-08T21:16:21,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:21,713 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02feb66403f44babbdfa40549c79c9fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733692579333 2024-12-08T21:16:21,713 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31e70a624f7429bad1b852685cd0c66, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733692581511 2024-12-08T21:16:21,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:21,714 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:21,716 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:21,716 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/B is initiating minor compaction (all files) 2024-12-08T21:16:21,716 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/B in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:21,716 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b248195fff074e329e5b90380b6881b0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/169e84455c774f10ba712229f49c12e0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e87c8f7a060a4208a961c757bfe3ee5f] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=35.3 K 2024-12-08T21:16:21,717 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b248195fff074e329e5b90380b6881b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692578990 2024-12-08T21:16:21,717 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 169e84455c774f10ba712229f49c12e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733692579333 2024-12-08T21:16:21,718 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e87c8f7a060a4208a961c757bfe3ee5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733692581517 2024-12-08T21:16:21,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:21,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:21,731 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#A#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:21,731 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/804ce5aa3a134dc682e2c7041abf8b67 is 50, key is test_row_0/A:col10/1733692581521/Put/seqid=0 2024-12-08T21:16:21,747 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:21,748 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/148946406a6349c9b63b6db80d9da210 is 50, key is test_row_0/B:col10/1733692581521/Put/seqid=0 2024-12-08T21:16:21,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-08T21:16:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:21,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:21,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:21,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:21,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:21,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:21,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:21,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742260_1436 (size=12207) 2024-12-08T21:16:21,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742261_1437 (size=12207) 2024-12-08T21:16:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692641792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/c19dcb5feef94e33b1c496e9bcc1ce46 is 50, key is test_row_0/A:col10/1733692581787/Put/seqid=0 2024-12-08T21:16:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692641808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692641809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692641810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742262_1438 (size=14341) 2024-12-08T21:16:21,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692641919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692641939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692641939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692641939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692642028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,037 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-08T21:16:22,089 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:22,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-08T21:16:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T21:16:22,091 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:22,092 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:22,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:22,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692642128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692642158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692642158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692642158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T21:16:22,199 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/804ce5aa3a134dc682e2c7041abf8b67 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/804ce5aa3a134dc682e2c7041abf8b67 2024-12-08T21:16:22,203 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/A of c482ca018e010a4a9e75b04b5a867252 into 804ce5aa3a134dc682e2c7041abf8b67(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:22,203 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:22,203 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/A, priority=13, startTime=1733692581710; duration=0sec 2024-12-08T21:16:22,203 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:22,203 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:22,203 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:22,213 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:22,217 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/C is initiating minor compaction (all files) 2024-12-08T21:16:22,217 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/C in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,217 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/148946406a6349c9b63b6db80d9da210 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/148946406a6349c9b63b6db80d9da210 2024-12-08T21:16:22,219 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/23bea873a4da4c88b077c3efe8346f82, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/cc2d6d5142de4475a91ce592ec6f58c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/054b50d6de904c4ba82fbc3d1178bff7] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=35.3 K 2024-12-08T21:16:22,220 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23bea873a4da4c88b077c3efe8346f82, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733692578990 2024-12-08T21:16:22,222 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc2d6d5142de4475a91ce592ec6f58c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1733692579333 2024-12-08T21:16:22,223 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054b50d6de904c4ba82fbc3d1178bff7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733692581517 2024-12-08T21:16:22,224 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/B of c482ca018e010a4a9e75b04b5a867252 into 148946406a6349c9b63b6db80d9da210(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:22,224 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:22,224 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/B, priority=13, startTime=1733692581714; duration=0sec 2024-12-08T21:16:22,226 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:22,226 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:22,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T21:16:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,253 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:22,254 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/8d5e79388e5e4c998ec85465337df52b is 50, key is test_row_0/C:col10/1733692581521/Put/seqid=0 2024-12-08T21:16:22,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/c19dcb5feef94e33b1c496e9bcc1ce46 2024-12-08T21:16:22,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742263_1439 (size=12207) 2024-12-08T21:16:22,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/afcab021f29249ad9a65b4c52209c4dd is 50, key is test_row_0/B:col10/1733692581787/Put/seqid=0 2024-12-08T21:16:22,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/8d5e79388e5e4c998ec85465337df52b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8d5e79388e5e4c998ec85465337df52b 2024-12-08T21:16:22,298 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/C of c482ca018e010a4a9e75b04b5a867252 into 8d5e79388e5e4c998ec85465337df52b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:22,298 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:22,298 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/C, priority=13, startTime=1733692581724; duration=0sec 2024-12-08T21:16:22,298 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:22,298 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:22,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742264_1440 (size=12001) 2024-12-08T21:16:22,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T21:16:22,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:22,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T21:16:22,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692642447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692642468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692642468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692642478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T21:16:22,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/afcab021f29249ad9a65b4c52209c4dd 2024-12-08T21:16:22,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T21:16:22,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:22,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f0300180a7744835a475189342bfdbb0 is 50, key is test_row_0/C:col10/1733692581787/Put/seqid=0 2024-12-08T21:16:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T21:16:22,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742265_1441 (size=12001) 2024-12-08T21:16:22,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T21:16:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:22,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:22,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692642958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692642977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:22,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692642987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:23,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692642998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:23,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:23,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:23,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:23,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T21:16:23,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:23,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:23,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:23,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:23,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f0300180a7744835a475189342bfdbb0 2024-12-08T21:16:23,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/c19dcb5feef94e33b1c496e9bcc1ce46 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c19dcb5feef94e33b1c496e9bcc1ce46 2024-12-08T21:16:23,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c19dcb5feef94e33b1c496e9bcc1ce46, entries=200, sequenceid=117, filesize=14.0 K 2024-12-08T21:16:23,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/afcab021f29249ad9a65b4c52209c4dd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/afcab021f29249ad9a65b4c52209c4dd 2024-12-08T21:16:23,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/afcab021f29249ad9a65b4c52209c4dd, entries=150, sequenceid=117, filesize=11.7 K 2024-12-08T21:16:23,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f0300180a7744835a475189342bfdbb0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f0300180a7744835a475189342bfdbb0 2024-12-08T21:16:23,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f0300180a7744835a475189342bfdbb0, entries=150, sequenceid=117, filesize=11.7 K 2024-12-08T21:16:23,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for c482ca018e010a4a9e75b04b5a867252 in 1345ms, sequenceid=117, compaction requested=false 2024-12-08T21:16:23,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:23,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-08T21:16:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:23,165 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T21:16:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:23,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/7e3a9158a7cc44df841518939ea7e94b is 50, key is test_row_0/A:col10/1733692581807/Put/seqid=0 2024-12-08T21:16:23,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742266_1442 (size=9657) 2024-12-08T21:16:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T21:16:23,582 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/7e3a9158a7cc44df841518939ea7e94b 2024-12-08T21:16:23,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/6c2daead9e064635bcd572fbe2cf08b7 is 50, key is test_row_0/B:col10/1733692581807/Put/seqid=0 2024-12-08T21:16:23,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742267_1443 (size=9657) 2024-12-08T21:16:23,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:24,002 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/6c2daead9e064635bcd572fbe2cf08b7 2024-12-08T21:16:24,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/4294363ab102467d94866a9eb16c755f is 50, key is test_row_0/C:col10/1733692581807/Put/seqid=0 2024-12-08T21:16:24,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742268_1444 (size=9657) 2024-12-08T21:16:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692584024, max timestamp=1733692584039Num uniques 859; 2024-12-08T21:16:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692584024, max timestamp=1733692584039Num uniques 859; 2024-12-08T21:16:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:16:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692584024, max timestamp=1733692584039Num uniques 859; 2024-12-08T21:16:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:16:24,041 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:16:24,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692644079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692584064, max timestamp=1733692584089Num uniques 1068; 2024-12-08T21:16:24,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-08T21:16:24,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733692584064, max timestamp=1733692584089Num uniques 1068; 2024-12-08T21:16:24,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-08T21:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692584064, max timestamp=1733692584089Num uniques 1068; 2024-12-08T21:16:24,094 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-08T21:16:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733692584128, max timestamp=1733692584140Num uniques 1077; 2024-12-08T21:16:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733692584128, max timestamp=1733692584140Num uniques 1127; 2024-12-08T21:16:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733692584128, max timestamp=1733692584140Num uniques 1077; 2024-12-08T21:16:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-12-08T21:16:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-08T21:16:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-08T21:16:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692644172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692644173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692644177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692644198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T21:16:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692644288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692644288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692644288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692644408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,427 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/4294363ab102467d94866a9eb16c755f 2024-12-08T21:16:24,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/7e3a9158a7cc44df841518939ea7e94b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/7e3a9158a7cc44df841518939ea7e94b 2024-12-08T21:16:24,435 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/7e3a9158a7cc44df841518939ea7e94b, entries=100, sequenceid=128, filesize=9.4 K 2024-12-08T21:16:24,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/6c2daead9e064635bcd572fbe2cf08b7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/6c2daead9e064635bcd572fbe2cf08b7 2024-12-08T21:16:24,439 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/6c2daead9e064635bcd572fbe2cf08b7, entries=100, sequenceid=128, filesize=9.4 K 2024-12-08T21:16:24,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/4294363ab102467d94866a9eb16c755f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/4294363ab102467d94866a9eb16c755f 2024-12-08T21:16:24,443 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/4294363ab102467d94866a9eb16c755f, entries=100, sequenceid=128, filesize=9.4 K 2024-12-08T21:16:24,444 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c482ca018e010a4a9e75b04b5a867252 in 1279ms, sequenceid=128, compaction requested=true 2024-12-08T21:16:24,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:24,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:24,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-08T21:16:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-08T21:16:24,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-08T21:16:24,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3530 sec 2024-12-08T21:16:24,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.3580 sec 2024-12-08T21:16:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:24,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692584170, max timestamp=1733692584507Num uniques 1136; 2024-12-08T21:16:24,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733692584170, max timestamp=1733692584507Num uniques 1136; 2024-12-08T21:16:24,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-08T21:16:24,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-08T21:16:24,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=69.33 KB heapSize=187.27 KB 2024-12-08T21:16:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:24,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d0fcdb50349d49b5aa2a2c967fdf5941 is 50, key is test_row_0/A:col10/1733692584171/Put/seqid=0 2024-12-08T21:16:24,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742269_1445 (size=14541) 2024-12-08T21:16:24,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692644576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692644577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692644583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692644698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692644698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692644700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692644718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692644908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692644908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692644908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:24,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d0fcdb50349d49b5aa2a2c967fdf5941 2024-12-08T21:16:24,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/e3b5c04f119f406e9d90a75684c811d0 is 50, key is test_row_0/B:col10/1733692584171/Put/seqid=0 2024-12-08T21:16:24,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742270_1446 (size=12151) 2024-12-08T21:16:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692645217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692645218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692645219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:25,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:25,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692645238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:25,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/e3b5c04f119f406e9d90a75684c811d0 2024-12-08T21:16:25,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/208e443339fc40f7add324dacc96e15a is 50, key is test_row_0/C:col10/1733692584171/Put/seqid=0 2024-12-08T21:16:25,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742271_1447 (size=12151) 2024-12-08T21:16:25,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692645728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:25,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:25,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692645738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692645738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:25,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/208e443339fc40f7add324dacc96e15a 2024-12-08T21:16:25,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d0fcdb50349d49b5aa2a2c967fdf5941 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d0fcdb50349d49b5aa2a2c967fdf5941 2024-12-08T21:16:25,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d0fcdb50349d49b5aa2a2c967fdf5941, entries=200, sequenceid=200, filesize=14.2 K 2024-12-08T21:16:25,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/e3b5c04f119f406e9d90a75684c811d0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e3b5c04f119f406e9d90a75684c811d0 2024-12-08T21:16:25,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e3b5c04f119f406e9d90a75684c811d0, entries=150, sequenceid=200, filesize=11.9 K 2024-12-08T21:16:25,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/208e443339fc40f7add324dacc96e15a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/208e443339fc40f7add324dacc96e15a 2024-12-08T21:16:25,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/208e443339fc40f7add324dacc96e15a, entries=150, sequenceid=200, filesize=11.9 K 2024-12-08T21:16:25,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~187.22 KB/191712, currentSize=114.05 KB/116790 for c482ca018e010a4a9e75b04b5a867252 in 1261ms, sequenceid=200, compaction requested=true 2024-12-08T21:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:25,772 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:25,772 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:25,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:25,773 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:25,773 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50746 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:25,773 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/B is initiating minor compaction (all files) 2024-12-08T21:16:25,773 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/A is initiating minor compaction (all files) 2024-12-08T21:16:25,774 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/A in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:25,774 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/B in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:25,774 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/804ce5aa3a134dc682e2c7041abf8b67, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c19dcb5feef94e33b1c496e9bcc1ce46, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/7e3a9158a7cc44df841518939ea7e94b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d0fcdb50349d49b5aa2a2c967fdf5941] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=49.6 K 2024-12-08T21:16:25,774 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/148946406a6349c9b63b6db80d9da210, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/afcab021f29249ad9a65b4c52209c4dd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/6c2daead9e064635bcd572fbe2cf08b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e3b5c04f119f406e9d90a75684c811d0] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=44.9 K 2024-12-08T21:16:25,774 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 148946406a6349c9b63b6db80d9da210, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733692581517 2024-12-08T21:16:25,774 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 804ce5aa3a134dc682e2c7041abf8b67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733692581517 2024-12-08T21:16:25,774 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting afcab021f29249ad9a65b4c52209c4dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733692581660 2024-12-08T21:16:25,774 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19dcb5feef94e33b1c496e9bcc1ce46, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733692581650 2024-12-08T21:16:25,775 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e3a9158a7cc44df841518939ea7e94b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733692581806 2024-12-08T21:16:25,775 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c2daead9e064635bcd572fbe2cf08b7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733692581806 2024-12-08T21:16:25,775 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e3b5c04f119f406e9d90a75684c811d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692584170 2024-12-08T21:16:25,775 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0fcdb50349d49b5aa2a2c967fdf5941, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692584170 2024-12-08T21:16:25,784 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:25,785 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/37989a3b842d4ad8bcaddd31f57959db is 50, key is test_row_0/B:col10/1733692584171/Put/seqid=0 2024-12-08T21:16:25,785 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:25,786 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/f7b1534d45c94d89957a66bbf2e26a06 is 50, key is test_row_0/A:col10/1733692584171/Put/seqid=0 2024-12-08T21:16:25,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742272_1448 (size=12493) 2024-12-08T21:16:25,802 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/f7b1534d45c94d89957a66bbf2e26a06 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7b1534d45c94d89957a66bbf2e26a06 2024-12-08T21:16:25,814 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/A of c482ca018e010a4a9e75b04b5a867252 into f7b1534d45c94d89957a66bbf2e26a06(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:25,814 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:25,814 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/A, priority=12, startTime=1733692585772; duration=0sec 2024-12-08T21:16:25,814 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:25,814 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:25,815 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:25,817 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:25,817 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/C is initiating minor compaction (all files) 2024-12-08T21:16:25,817 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/C in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:25,817 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8d5e79388e5e4c998ec85465337df52b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f0300180a7744835a475189342bfdbb0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/4294363ab102467d94866a9eb16c755f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/208e443339fc40f7add324dacc96e15a] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=44.9 K 2024-12-08T21:16:25,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742273_1449 (size=12493) 2024-12-08T21:16:25,818 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d5e79388e5e4c998ec85465337df52b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1733692581517 2024-12-08T21:16:25,819 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0300180a7744835a475189342bfdbb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733692581660 2024-12-08T21:16:25,819 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4294363ab102467d94866a9eb16c755f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733692581806 2024-12-08T21:16:25,819 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 208e443339fc40f7add324dacc96e15a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692584170 2024-12-08T21:16:25,822 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/37989a3b842d4ad8bcaddd31f57959db as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/37989a3b842d4ad8bcaddd31f57959db 2024-12-08T21:16:25,827 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/B of c482ca018e010a4a9e75b04b5a867252 into 37989a3b842d4ad8bcaddd31f57959db(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:25,827 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:25,827 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/B, priority=12, startTime=1733692585772; duration=0sec 2024-12-08T21:16:25,827 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:25,827 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:25,832 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#C#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:25,832 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a5f483d20efb42a69894daa17e0cd474 is 50, key is test_row_0/C:col10/1733692584171/Put/seqid=0 2024-12-08T21:16:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742274_1450 (size=12493) 2024-12-08T21:16:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:26,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T21:16:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:26,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:26,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/46d6247385204365a164a11c49b3629e is 50, key is test_row_0/A:col10/1733692584573/Put/seqid=0 2024-12-08T21:16:26,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742275_1451 (size=14541) 2024-12-08T21:16:26,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692646106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692646217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-08T21:16:26,258 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:26,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-08T21:16:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T21:16:26,260 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:26,261 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:26,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:26,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692646258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,264 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,287 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a5f483d20efb42a69894daa17e0cd474 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a5f483d20efb42a69894daa17e0cd474 2024-12-08T21:16:26,292 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/C of c482ca018e010a4a9e75b04b5a867252 into a5f483d20efb42a69894daa17e0cd474(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:26,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:26,292 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/C, priority=12, startTime=1733692585772; duration=0sec 2024-12-08T21:16:26,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:26,292 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T21:16:26,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T21:16:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692646428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/46d6247385204365a164a11c49b3629e 2024-12-08T21:16:26,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/3afd39c7c68b43959e12e69256a2e55e is 50, key is test_row_0/B:col10/1733692584573/Put/seqid=0 2024-12-08T21:16:26,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742276_1452 (size=12151) 2024-12-08T21:16:26,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T21:16:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T21:16:26,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T21:16:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:26,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692646738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692646748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,755 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692646758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692646758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:26,766 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:26,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,766 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:26,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T21:16:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:26,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:26,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/3afd39c7c68b43959e12e69256a2e55e 2024-12-08T21:16:26,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/00f4336447a94e749b6fc98ff39e3963 is 50, key is test_row_0/C:col10/1733692584573/Put/seqid=0 2024-12-08T21:16:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T21:16:26,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742277_1453 (size=12151) 2024-12-08T21:16:27,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T21:16:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:27,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:27,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:27,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:27,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T21:16:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:27,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:27,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692647248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:27,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/00f4336447a94e749b6fc98ff39e3963 2024-12-08T21:16:27,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/46d6247385204365a164a11c49b3629e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/46d6247385204365a164a11c49b3629e 2024-12-08T21:16:27,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/46d6247385204365a164a11c49b3629e, entries=200, sequenceid=223, filesize=14.2 K 2024-12-08T21:16:27,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/3afd39c7c68b43959e12e69256a2e55e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/3afd39c7c68b43959e12e69256a2e55e 2024-12-08T21:16:27,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/3afd39c7c68b43959e12e69256a2e55e, entries=150, sequenceid=223, filesize=11.9 K 2024-12-08T21:16:27,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/00f4336447a94e749b6fc98ff39e3963 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/00f4336447a94e749b6fc98ff39e3963 2024-12-08T21:16:27,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/00f4336447a94e749b6fc98ff39e3963, entries=150, sequenceid=223, filesize=11.9 K 2024-12-08T21:16:27,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for c482ca018e010a4a9e75b04b5a867252 in 1244ms, sequenceid=223, compaction requested=false 2024-12-08T21:16:27,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:27,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-08T21:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:27,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T21:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:27,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/a28f62660daa465ca2e8cdd24a9efcdd is 50, key is test_row_0/A:col10/1733692586089/Put/seqid=0 2024-12-08T21:16:27,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742278_1454 (size=12151) 2024-12-08T21:16:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T21:16:27,743 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/a28f62660daa465ca2e8cdd24a9efcdd 2024-12-08T21:16:27,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/c8b60721dfa9490cae1c35c97e1edb5f is 50, key is test_row_0/B:col10/1733692586089/Put/seqid=0 2024-12-08T21:16:27,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742279_1455 (size=12151) 2024-12-08T21:16:28,155 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/c8b60721dfa9490cae1c35c97e1edb5f 2024-12-08T21:16:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f4b06b0e5aaa41308606b2daeed5cb1d is 50, key is test_row_0/C:col10/1733692586089/Put/seqid=0 2024-12-08T21:16:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742280_1456 (size=12151) 2024-12-08T21:16:28,182 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f4b06b0e5aaa41308606b2daeed5cb1d 2024-12-08T21:16:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/a28f62660daa465ca2e8cdd24a9efcdd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a28f62660daa465ca2e8cdd24a9efcdd 2024-12-08T21:16:28,192 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a28f62660daa465ca2e8cdd24a9efcdd, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T21:16:28,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/c8b60721dfa9490cae1c35c97e1edb5f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/c8b60721dfa9490cae1c35c97e1edb5f 2024-12-08T21:16:28,201 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/c8b60721dfa9490cae1c35c97e1edb5f, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T21:16:28,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f4b06b0e5aaa41308606b2daeed5cb1d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f4b06b0e5aaa41308606b2daeed5cb1d 2024-12-08T21:16:28,207 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f4b06b0e5aaa41308606b2daeed5cb1d, entries=150, sequenceid=235, filesize=11.9 K 2024-12-08T21:16:28,207 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for c482ca018e010a4a9e75b04b5a867252 in 877ms, sequenceid=235, compaction requested=true 2024-12-08T21:16:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-08T21:16:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-08T21:16:28,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-08T21:16:28,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9490 sec 2024-12-08T21:16:28,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9530 sec 2024-12-08T21:16:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:28,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:16:28,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:28,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:28,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:28,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:28,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:28,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:28,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/953cb3449a60410dbb2205eee5503058 is 50, key is test_row_0/A:col10/1733692588290/Put/seqid=0 2024-12-08T21:16:28,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742281_1457 (size=14541) 2024-12-08T21:16:28,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/953cb3449a60410dbb2205eee5503058 2024-12-08T21:16:28,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/acae0d5c1e364be6b5b963b8f6dd14fd is 50, key is test_row_0/B:col10/1733692588290/Put/seqid=0 2024-12-08T21:16:28,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742282_1458 (size=12151) 2024-12-08T21:16:28,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/acae0d5c1e364be6b5b963b8f6dd14fd 2024-12-08T21:16:28,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/24360375fa7a441f90ee7c019f406169 is 50, key is test_row_0/C:col10/1733692588290/Put/seqid=0 2024-12-08T21:16:28,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742283_1459 (size=12151) 2024-12-08T21:16:28,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/24360375fa7a441f90ee7c019f406169 2024-12-08T21:16:28,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/953cb3449a60410dbb2205eee5503058 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/953cb3449a60410dbb2205eee5503058 2024-12-08T21:16:28,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/953cb3449a60410dbb2205eee5503058, entries=200, sequenceid=245, filesize=14.2 K 2024-12-08T21:16:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/acae0d5c1e364be6b5b963b8f6dd14fd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/acae0d5c1e364be6b5b963b8f6dd14fd 2024-12-08T21:16:28,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/acae0d5c1e364be6b5b963b8f6dd14fd, entries=150, sequenceid=245, filesize=11.9 K 2024-12-08T21:16:28,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/24360375fa7a441f90ee7c019f406169 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/24360375fa7a441f90ee7c019f406169 2024-12-08T21:16:28,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/24360375fa7a441f90ee7c019f406169, entries=150, sequenceid=245, filesize=11.9 K 2024-12-08T21:16:28,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for c482ca018e010a4a9e75b04b5a867252 in 71ms, sequenceid=245, compaction requested=true 2024-12-08T21:16:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:28,362 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:28,362 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:28,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:28,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:28,363 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:28,363 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:28,363 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/A is initiating minor compaction (all files) 2024-12-08T21:16:28,363 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/B is initiating minor compaction (all files) 2024-12-08T21:16:28,363 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/A in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,363 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/B in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,364 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7b1534d45c94d89957a66bbf2e26a06, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/46d6247385204365a164a11c49b3629e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a28f62660daa465ca2e8cdd24a9efcdd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/953cb3449a60410dbb2205eee5503058] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=52.5 K 2024-12-08T21:16:28,364 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/37989a3b842d4ad8bcaddd31f57959db, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/3afd39c7c68b43959e12e69256a2e55e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/c8b60721dfa9490cae1c35c97e1edb5f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/acae0d5c1e364be6b5b963b8f6dd14fd] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=47.8 K 2024-12-08T21:16:28,364 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 37989a3b842d4ad8bcaddd31f57959db, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692584170 2024-12-08T21:16:28,364 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 3afd39c7c68b43959e12e69256a2e55e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733692584573 2024-12-08T21:16:28,365 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b60721dfa9490cae1c35c97e1edb5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733692586089 2024-12-08T21:16:28,365 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting acae0d5c1e364be6b5b963b8f6dd14fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733692588283 2024-12-08T21:16:28,365 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b1534d45c94d89957a66bbf2e26a06, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692584170 2024-12-08T21:16:28,366 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d6247385204365a164a11c49b3629e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733692584573 2024-12-08T21:16:28,366 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28f62660daa465ca2e8cdd24a9efcdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733692586089 2024-12-08T21:16:28,366 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 953cb3449a60410dbb2205eee5503058, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733692588277 2024-12-08T21:16:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:28,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T21:16:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:28,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:28,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/f62de546ce7b449b8e41481d8d9444b5 is 50, key is test_row_0/A:col10/1733692588363/Put/seqid=0 2024-12-08T21:16:28,378 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:28,378 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/b5a74149da9047d0b468f69f841c5658 is 50, key is test_row_0/B:col10/1733692588290/Put/seqid=0 2024-12-08T21:16:28,388 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:28,388 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/0f196adb60c446d9816d1b9f5e943325 is 50, key is test_row_0/A:col10/1733692588290/Put/seqid=0 2024-12-08T21:16:28,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742284_1460 (size=14741) 2024-12-08T21:16:28,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742285_1461 (size=12629) 2024-12-08T21:16:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-08T21:16:28,408 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:28,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-08T21:16:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T21:16:28,412 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:28,413 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:28,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:28,414 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/b5a74149da9047d0b468f69f841c5658 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b5a74149da9047d0b468f69f841c5658 2024-12-08T21:16:28,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742286_1462 (size=12629) 2024-12-08T21:16:28,422 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/B of c482ca018e010a4a9e75b04b5a867252 into b5a74149da9047d0b468f69f841c5658(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:28,422 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:28,422 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/B, priority=12, startTime=1733692588362; duration=0sec 2024-12-08T21:16:28,422 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:28,422 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:28,422 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:28,423 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:28,423 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/C is initiating minor compaction (all files) 2024-12-08T21:16:28,423 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/C in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,423 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a5f483d20efb42a69894daa17e0cd474, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/00f4336447a94e749b6fc98ff39e3963, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f4b06b0e5aaa41308606b2daeed5cb1d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/24360375fa7a441f90ee7c019f406169] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=47.8 K 2024-12-08T21:16:28,423 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a5f483d20efb42a69894daa17e0cd474, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1733692584170 2024-12-08T21:16:28,423 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f4336447a94e749b6fc98ff39e3963, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733692584573 2024-12-08T21:16:28,424 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f4b06b0e5aaa41308606b2daeed5cb1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733692586089 2024-12-08T21:16:28,425 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 24360375fa7a441f90ee7c019f406169, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733692588283 2024-12-08T21:16:28,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692648413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692648425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,432 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#C#compaction#398 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:28,433 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/fcf6d735fb0e4abaa01b34c030a7f43d is 50, key is test_row_0/C:col10/1733692588290/Put/seqid=0 2024-12-08T21:16:28,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742287_1463 (size=12629) 2024-12-08T21:16:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T21:16:28,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692648527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692648538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T21:16:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T21:16:28,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692648738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692648758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692648768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,772 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692648778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,781 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692648788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:28,792 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:28,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:28,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/f62de546ce7b449b8e41481d8d9444b5 2024-12-08T21:16:28,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/a6be96e81b1442bf9361f180fed95723 is 50, key is test_row_0/B:col10/1733692588363/Put/seqid=0 2024-12-08T21:16:28,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742288_1464 (size=12301) 2024-12-08T21:16:28,828 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/0f196adb60c446d9816d1b9f5e943325 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/0f196adb60c446d9816d1b9f5e943325 2024-12-08T21:16:28,833 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/A of c482ca018e010a4a9e75b04b5a867252 into 0f196adb60c446d9816d1b9f5e943325(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:28,833 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:28,833 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/A, priority=12, startTime=1733692588362; duration=0sec 2024-12-08T21:16:28,833 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:28,833 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:28,845 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/fcf6d735fb0e4abaa01b34c030a7f43d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/fcf6d735fb0e4abaa01b34c030a7f43d 2024-12-08T21:16:28,850 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/C of c482ca018e010a4a9e75b04b5a867252 into fcf6d735fb0e4abaa01b34c030a7f43d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:28,850 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:28,851 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/C, priority=12, startTime=1733692588362; duration=0sec 2024-12-08T21:16:28,851 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:28,851 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:28,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T21:16:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:28,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T21:16:29,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T21:16:29,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692649058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:29,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692649068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:29,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T21:16:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/a6be96e81b1442bf9361f180fed95723 2024-12-08T21:16:29,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a3c1bda7ff0b4e138f9e7fef71e0757e is 50, key is test_row_0/C:col10/1733692588363/Put/seqid=0 2024-12-08T21:16:29,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742289_1465 (size=12301) 2024-12-08T21:16:29,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a3c1bda7ff0b4e138f9e7fef71e0757e 2024-12-08T21:16:29,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/f62de546ce7b449b8e41481d8d9444b5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f62de546ce7b449b8e41481d8d9444b5 2024-12-08T21:16:29,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f62de546ce7b449b8e41481d8d9444b5, entries=200, sequenceid=264, filesize=14.4 K 2024-12-08T21:16:29,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/a6be96e81b1442bf9361f180fed95723 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a6be96e81b1442bf9361f180fed95723 2024-12-08T21:16:29,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a6be96e81b1442bf9361f180fed95723, entries=150, sequenceid=264, filesize=12.0 K 2024-12-08T21:16:29,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a3c1bda7ff0b4e138f9e7fef71e0757e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a3c1bda7ff0b4e138f9e7fef71e0757e 2024-12-08T21:16:29,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a3c1bda7ff0b4e138f9e7fef71e0757e, entries=150, sequenceid=264, filesize=12.0 K 2024-12-08T21:16:29,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c482ca018e010a4a9e75b04b5a867252 in 930ms, sequenceid=264, compaction requested=false 2024-12-08T21:16:29,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:29,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-08T21:16:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,338 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:16:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:29,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:29,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:29,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:29,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:29,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/a558d5ab260444b09d7531c2d7480452 is 50, key is test_row_0/A:col10/1733692588411/Put/seqid=0 2024-12-08T21:16:29,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742290_1466 (size=12301) 2024-12-08T21:16:29,393 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/a558d5ab260444b09d7531c2d7480452 2024-12-08T21:16:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/144d6dbc45bf47f9adb85aa7a092090a is 50, key is test_row_0/B:col10/1733692588411/Put/seqid=0 2024-12-08T21:16:29,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742291_1467 (size=12301) 2024-12-08T21:16:29,412 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/144d6dbc45bf47f9adb85aa7a092090a 2024-12-08T21:16:29,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/2759280924f74bf4a45fa5e838569723 is 50, key is test_row_0/C:col10/1733692588411/Put/seqid=0 2024-12-08T21:16:29,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742292_1468 (size=12301) 2024-12-08T21:16:29,471 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/2759280924f74bf4a45fa5e838569723 2024-12-08T21:16:29,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/a558d5ab260444b09d7531c2d7480452 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a558d5ab260444b09d7531c2d7480452 2024-12-08T21:16:29,483 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a558d5ab260444b09d7531c2d7480452, entries=150, sequenceid=280, filesize=12.0 K 2024-12-08T21:16:29,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/144d6dbc45bf47f9adb85aa7a092090a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/144d6dbc45bf47f9adb85aa7a092090a 2024-12-08T21:16:29,490 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/144d6dbc45bf47f9adb85aa7a092090a, entries=150, sequenceid=280, filesize=12.0 K 2024-12-08T21:16:29,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/2759280924f74bf4a45fa5e838569723 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/2759280924f74bf4a45fa5e838569723 2024-12-08T21:16:29,499 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/2759280924f74bf4a45fa5e838569723, entries=150, sequenceid=280, filesize=12.0 K 2024-12-08T21:16:29,502 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for c482ca018e010a4a9e75b04b5a867252 in 164ms, sequenceid=280, compaction requested=true 2024-12-08T21:16:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-08T21:16:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-08T21:16:29,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-08T21:16:29,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-12-08T21:16:29,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.1000 sec 2024-12-08T21:16:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-08T21:16:29,548 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:29,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-08T21:16:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T21:16:29,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:29,552 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:29,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:29,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:16:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:29,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/61d4a1f5222e4a4a8503943c1b54c9dd is 50, key is test_row_0/A:col10/1733692589596/Put/seqid=0 2024-12-08T21:16:29,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742293_1469 (size=12297) 2024-12-08T21:16:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T21:16:29,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692589653, max timestamp=1733692589666Num uniques 859; 2024-12-08T21:16:29,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692589653, max timestamp=1733692589666Num uniques 859; 2024-12-08T21:16:29,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:29,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:29,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692589653, max timestamp=1733692589666Num uniques 859; 2024-12-08T21:16:29,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:29,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:29,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692649761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:29,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692649768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:29,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:29,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T21:16:29,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692649878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:29,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692649878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:30,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/61d4a1f5222e4a4a8503943c1b54c9dd 2024-12-08T21:16:30,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/f7427b01cc5d42aa864ca410e37dfb67 is 50, key is test_row_0/B:col10/1733692589596/Put/seqid=0 2024-12-08T21:16:30,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742294_1470 (size=9857) 2024-12-08T21:16:30,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692650088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:30,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692650088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:30,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T21:16:30,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:30,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692650398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:30,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692650407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:30,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/f7427b01cc5d42aa864ca410e37dfb67 2024-12-08T21:16:30,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/62c6df2906594de282b6d514b99f976a is 50, key is test_row_0/C:col10/1733692589596/Put/seqid=0 2024-12-08T21:16:30,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742295_1471 (size=9857) 2024-12-08T21:16:30,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:30,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T21:16:30,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/62c6df2906594de282b6d514b99f976a 2024-12-08T21:16:30,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/61d4a1f5222e4a4a8503943c1b54c9dd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/61d4a1f5222e4a4a8503943c1b54c9dd 2024-12-08T21:16:30,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/61d4a1f5222e4a4a8503943c1b54c9dd, entries=150, sequenceid=290, filesize=12.0 K 2024-12-08T21:16:30,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/f7427b01cc5d42aa864ca410e37dfb67 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/f7427b01cc5d42aa864ca410e37dfb67 2024-12-08T21:16:30,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/f7427b01cc5d42aa864ca410e37dfb67, entries=100, sequenceid=290, filesize=9.6 K 2024-12-08T21:16:30,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/62c6df2906594de282b6d514b99f976a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/62c6df2906594de282b6d514b99f976a 2024-12-08T21:16:30,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/62c6df2906594de282b6d514b99f976a, entries=100, sequenceid=290, filesize=9.6 K 2024-12-08T21:16:30,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c482ca018e010a4a9e75b04b5a867252 in 1259ms, sequenceid=290, compaction requested=true 2024-12-08T21:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:30,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:30,858 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:30,858 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:30,858 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51968 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:30,858 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:30,858 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/A is initiating minor compaction (all files) 2024-12-08T21:16:30,858 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/B is initiating minor compaction (all files) 2024-12-08T21:16:30,858 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/A in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,858 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/B in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,858 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/0f196adb60c446d9816d1b9f5e943325, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f62de546ce7b449b8e41481d8d9444b5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a558d5ab260444b09d7531c2d7480452, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/61d4a1f5222e4a4a8503943c1b54c9dd] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=50.8 K 2024-12-08T21:16:30,858 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b5a74149da9047d0b468f69f841c5658, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a6be96e81b1442bf9361f180fed95723, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/144d6dbc45bf47f9adb85aa7a092090a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/f7427b01cc5d42aa864ca410e37dfb67] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=46.0 K 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a74149da9047d0b468f69f841c5658, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733692588283 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f196adb60c446d9816d1b9f5e943325, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733692588283 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a6be96e81b1442bf9361f180fed95723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733692588346 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting f62de546ce7b449b8e41481d8d9444b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733692588346 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 144d6dbc45bf47f9adb85aa7a092090a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733692588383 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a558d5ab260444b09d7531c2d7480452, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733692588383 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f7427b01cc5d42aa864ca410e37dfb67, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733692589594 2024-12-08T21:16:30,859 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61d4a1f5222e4a4a8503943c1b54c9dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733692589591 2024-12-08T21:16:30,868 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#B#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:30,868 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/008057e315fc497aa35629fbed5a9586 is 50, key is test_row_0/B:col10/1733692589596/Put/seqid=0 2024-12-08T21:16:30,869 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:30,869 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/6553fd82cb02420b992bf8d970ce0b10 is 50, key is test_row_0/A:col10/1733692589596/Put/seqid=0 2024-12-08T21:16:30,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742296_1472 (size=12915) 2024-12-08T21:16:30,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742297_1473 (size=12915) 2024-12-08T21:16:30,881 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/6553fd82cb02420b992bf8d970ce0b10 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/6553fd82cb02420b992bf8d970ce0b10 2024-12-08T21:16:30,884 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/A of c482ca018e010a4a9e75b04b5a867252 into 6553fd82cb02420b992bf8d970ce0b10(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:30,884 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:30,884 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/A, priority=12, startTime=1733692590857; duration=0sec 2024-12-08T21:16:30,884 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:30,884 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:30,884 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:30,885 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47088 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:30,885 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/C is initiating minor compaction (all files) 2024-12-08T21:16:30,885 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/C in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,885 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/fcf6d735fb0e4abaa01b34c030a7f43d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a3c1bda7ff0b4e138f9e7fef71e0757e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/2759280924f74bf4a45fa5e838569723, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/62c6df2906594de282b6d514b99f976a] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=46.0 K 2024-12-08T21:16:30,886 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf6d735fb0e4abaa01b34c030a7f43d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733692588283 2024-12-08T21:16:30,886 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3c1bda7ff0b4e138f9e7fef71e0757e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733692588346 2024-12-08T21:16:30,886 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2759280924f74bf4a45fa5e838569723, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733692588383 2024-12-08T21:16:30,886 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c6df2906594de282b6d514b99f976a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733692589594 2024-12-08T21:16:30,892 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#C#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:30,893 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/024978f63e3141d78712cb9ea1b8e932 is 50, key is test_row_0/C:col10/1733692589596/Put/seqid=0 2024-12-08T21:16:30,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742298_1474 (size=12915) 2024-12-08T21:16:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:30,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-08T21:16:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:30,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:30,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:30,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:30,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/381f16352c474732b6e763a1fe1589f8 is 50, key is test_row_0/A:col10/1733692589729/Put/seqid=0 2024-12-08T21:16:30,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742299_1475 (size=14741) 2024-12-08T21:16:30,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692650946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:30,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692650946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692651058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692651058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:31,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:31,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:31,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692651267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:31,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692651268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:31,275 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/008057e315fc497aa35629fbed5a9586 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/008057e315fc497aa35629fbed5a9586 2024-12-08T21:16:31,279 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/B of c482ca018e010a4a9e75b04b5a867252 into 008057e315fc497aa35629fbed5a9586(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:31,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:31,279 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/B, priority=12, startTime=1733692590858; duration=0sec 2024-12-08T21:16:31,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:31,279 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:31,300 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/024978f63e3141d78712cb9ea1b8e932 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/024978f63e3141d78712cb9ea1b8e932 2024-12-08T21:16:31,304 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/C of c482ca018e010a4a9e75b04b5a867252 into 024978f63e3141d78712cb9ea1b8e932(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:31,304 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:31,304 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/C, priority=12, startTime=1733692590858; duration=0sec 2024-12-08T21:16:31,304 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:31,304 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:31,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/381f16352c474732b6e763a1fe1589f8 2024-12-08T21:16:31,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/a3dcb80f364240db9feb03676a1ab78f is 50, key is test_row_0/B:col10/1733692589729/Put/seqid=0 2024-12-08T21:16:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742300_1476 (size=12301) 2024-12-08T21:16:31,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:31,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:31,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692651578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:31,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692651588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:31,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T21:16:31,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/a3dcb80f364240db9feb03676a1ab78f 2024-12-08T21:16:31,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/78e45901c82d49059f0f7fb2d01a7e2e is 50, key is test_row_0/C:col10/1733692589729/Put/seqid=0 2024-12-08T21:16:31,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742301_1477 (size=12301) 2024-12-08T21:16:31,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:31,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692652097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692652097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:32,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:32,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:32,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/78e45901c82d49059f0f7fb2d01a7e2e 2024-12-08T21:16:32,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/381f16352c474732b6e763a1fe1589f8 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/381f16352c474732b6e763a1fe1589f8 2024-12-08T21:16:32,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/381f16352c474732b6e763a1fe1589f8, entries=200, sequenceid=329, filesize=14.4 K 2024-12-08T21:16:32,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/a3dcb80f364240db9feb03676a1ab78f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a3dcb80f364240db9feb03676a1ab78f 2024-12-08T21:16:32,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a3dcb80f364240db9feb03676a1ab78f, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T21:16:32,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/78e45901c82d49059f0f7fb2d01a7e2e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/78e45901c82d49059f0f7fb2d01a7e2e 2024-12-08T21:16:32,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/78e45901c82d49059f0f7fb2d01a7e2e, entries=150, sequenceid=329, filesize=12.0 K 2024-12-08T21:16:32,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for c482ca018e010a4a9e75b04b5a867252 in 1253ms, sequenceid=329, compaction requested=false 2024-12-08T21:16:32,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:32,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-08T21:16:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:32,302 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-08T21:16:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:32,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:32,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:32,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/10a1dd8aba2c4368b4aeb0aec79bbd7f is 50, key is test_row_0/A:col10/1733692590936/Put/seqid=0 2024-12-08T21:16:32,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742302_1478 (size=12301) 2024-12-08T21:16:32,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:16:32,710 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/10a1dd8aba2c4368b4aeb0aec79bbd7f 2024-12-08T21:16:32,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/26b59fc648af45e09623a64294eece05 is 50, key is test_row_0/B:col10/1733692590936/Put/seqid=0 2024-12-08T21:16:32,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742303_1479 (size=12301) 2024-12-08T21:16:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:32,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692592821, max timestamp=1733692592844Num uniques 859; 2024-12-08T21:16:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:16:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692592821, max timestamp=1733692592844Num uniques 859; 2024-12-08T21:16:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733692592821, max timestamp=1733692592844Num uniques 859; 2024-12-08T21:16:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:16:32,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-08T21:16:32,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692652877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:32,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692652883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692652885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:32,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692652988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:32,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692652988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:32,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692653007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692653107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,115 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,120 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/26b59fc648af45e09623a64294eece05 2024-12-08T21:16:33,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a6c802d6390340dea2e92fdd6eb6c6ae is 50, key is test_row_0/C:col10/1733692590936/Put/seqid=0 2024-12-08T21:16:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692653117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,126 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742304_1480 (size=12301) 2024-12-08T21:16:33,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692653198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692653198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692653217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692653508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692653508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,529 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a6c802d6390340dea2e92fdd6eb6c6ae 2024-12-08T21:16:33,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/10a1dd8aba2c4368b4aeb0aec79bbd7f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/10a1dd8aba2c4368b4aeb0aec79bbd7f 2024-12-08T21:16:33,535 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/10a1dd8aba2c4368b4aeb0aec79bbd7f, entries=150, sequenceid=340, filesize=12.0 K 2024-12-08T21:16:33,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/26b59fc648af45e09623a64294eece05 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/26b59fc648af45e09623a64294eece05 2024-12-08T21:16:33,538 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/26b59fc648af45e09623a64294eece05, entries=150, sequenceid=340, filesize=12.0 K 2024-12-08T21:16:33,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a6c802d6390340dea2e92fdd6eb6c6ae as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a6c802d6390340dea2e92fdd6eb6c6ae 2024-12-08T21:16:33,541 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a6c802d6390340dea2e92fdd6eb6c6ae, entries=150, sequenceid=340, filesize=12.0 K 2024-12-08T21:16:33,542 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for c482ca018e010a4a9e75b04b5a867252 in 1240ms, sequenceid=340, compaction requested=true 2024-12-08T21:16:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-08T21:16:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-08T21:16:33,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-08T21:16:33,544 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9900 sec 2024-12-08T21:16:33,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 3.9950 sec 2024-12-08T21:16:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:33,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-08T21:16:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:33,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:33,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:33,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d1deceb825474159975e9512808c8d50 is 50, key is test_row_0/A:col10/1733692592882/Put/seqid=0 2024-12-08T21:16:33,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742305_1481 (size=14741) 2024-12-08T21:16:33,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692653575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692653687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-08T21:16:33,708 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:33,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-08T21:16:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T21:16:33,710 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:33,710 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:33,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T21:16:33,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:33,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:33,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:33,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692653897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:33,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d1deceb825474159975e9512808c8d50 2024-12-08T21:16:33,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/df5c57aad3074ef899539950ad4a7e25 is 50, key is test_row_0/B:col10/1733692592882/Put/seqid=0 2024-12-08T21:16:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742306_1482 (size=12301) 2024-12-08T21:16:34,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:34,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692654018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692654018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T21:16:34,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692654208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:34,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T21:16:34,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/df5c57aad3074ef899539950ad4a7e25 2024-12-08T21:16:34,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f1e5bdcc9039483b8a549f5c9956d292 is 50, key is test_row_0/C:col10/1733692592882/Put/seqid=0 2024-12-08T21:16:34,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742307_1483 (size=12301) 2024-12-08T21:16:34,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:34,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:34,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692654728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:34,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:34,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f1e5bdcc9039483b8a549f5c9956d292 2024-12-08T21:16:34,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:34,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:34,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:34,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d1deceb825474159975e9512808c8d50 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1deceb825474159975e9512808c8d50 2024-12-08T21:16:34,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1deceb825474159975e9512808c8d50, entries=200, sequenceid=377, filesize=14.4 K 2024-12-08T21:16:34,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/df5c57aad3074ef899539950ad4a7e25 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/df5c57aad3074ef899539950ad4a7e25 2024-12-08T21:16:34,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/df5c57aad3074ef899539950ad4a7e25, entries=150, sequenceid=377, filesize=12.0 K 2024-12-08T21:16:34,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/f1e5bdcc9039483b8a549f5c9956d292 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f1e5bdcc9039483b8a549f5c9956d292 2024-12-08T21:16:34,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f1e5bdcc9039483b8a549f5c9956d292, entries=150, sequenceid=377, filesize=12.0 K 2024-12-08T21:16:34,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for c482ca018e010a4a9e75b04b5a867252 in 1243ms, sequenceid=377, compaction requested=true 2024-12-08T21:16:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:34,791 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:34,791 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:34,792 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:34,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:34,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/A is initiating minor compaction (all files) 2024-12-08T21:16:34,792 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/B is initiating minor compaction (all files) 2024-12-08T21:16:34,792 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/B in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,792 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/A in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,792 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/008057e315fc497aa35629fbed5a9586, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a3dcb80f364240db9feb03676a1ab78f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/26b59fc648af45e09623a64294eece05, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/df5c57aad3074ef899539950ad4a7e25] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=48.7 K 2024-12-08T21:16:34,792 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/6553fd82cb02420b992bf8d970ce0b10, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/381f16352c474732b6e763a1fe1589f8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/10a1dd8aba2c4368b4aeb0aec79bbd7f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1deceb825474159975e9512808c8d50] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=53.4 K 2024-12-08T21:16:34,792 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 008057e315fc497aa35629fbed5a9586, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733692588424 2024-12-08T21:16:34,792 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6553fd82cb02420b992bf8d970ce0b10, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733692588424 2024-12-08T21:16:34,793 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a3dcb80f364240db9feb03676a1ab78f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733692589729 2024-12-08T21:16:34,793 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381f16352c474732b6e763a1fe1589f8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733692589729 2024-12-08T21:16:34,793 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b59fc648af45e09623a64294eece05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733692590936 2024-12-08T21:16:34,793 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10a1dd8aba2c4368b4aeb0aec79bbd7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733692590936 2024-12-08T21:16:34,793 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting df5c57aad3074ef899539950ad4a7e25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733692592882 2024-12-08T21:16:34,793 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1deceb825474159975e9512808c8d50, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733692592876 2024-12-08T21:16:34,800 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:34,801 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/bcc2a64bd4654f319f047c6df092cddf is 50, key is test_row_0/B:col10/1733692592882/Put/seqid=0 2024-12-08T21:16:34,801 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:34,802 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/f7a28660ec94448d8a04ee91f2977bd3 is 50, key is test_row_0/A:col10/1733692592882/Put/seqid=0 2024-12-08T21:16:34,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742308_1484 (size=13051) 2024-12-08T21:16:34,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742309_1485 (size=13051) 2024-12-08T21:16:34,811 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/bcc2a64bd4654f319f047c6df092cddf as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/bcc2a64bd4654f319f047c6df092cddf 2024-12-08T21:16:34,812 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/f7a28660ec94448d8a04ee91f2977bd3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7a28660ec94448d8a04ee91f2977bd3 2024-12-08T21:16:34,816 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/B of c482ca018e010a4a9e75b04b5a867252 into bcc2a64bd4654f319f047c6df092cddf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:34,816 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:34,816 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/B, priority=12, startTime=1733692594791; duration=0sec 2024-12-08T21:16:34,816 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:34,816 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:34,816 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:16:34,816 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/A of c482ca018e010a4a9e75b04b5a867252 into f7a28660ec94448d8a04ee91f2977bd3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:34,816 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:34,816 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/A, priority=12, startTime=1733692594791; duration=0sec 2024-12-08T21:16:34,816 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:34,816 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:34,817 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:16:34,817 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/C is initiating minor compaction (all files) 2024-12-08T21:16:34,817 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/C in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,817 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/024978f63e3141d78712cb9ea1b8e932, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/78e45901c82d49059f0f7fb2d01a7e2e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a6c802d6390340dea2e92fdd6eb6c6ae, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f1e5bdcc9039483b8a549f5c9956d292] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=48.7 K 2024-12-08T21:16:34,817 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 024978f63e3141d78712cb9ea1b8e932, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733692588424 2024-12-08T21:16:34,817 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e45901c82d49059f0f7fb2d01a7e2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1733692589729 2024-12-08T21:16:34,817 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c802d6390340dea2e92fdd6eb6c6ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733692590936 2024-12-08T21:16:34,818 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e5bdcc9039483b8a549f5c9956d292, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733692592882 2024-12-08T21:16:34,823 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#C#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:34,823 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/b6c0c3f31033450798a310a8beaa7bf7 is 50, key is test_row_0/C:col10/1733692592882/Put/seqid=0 2024-12-08T21:16:34,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742310_1486 (size=13051) 2024-12-08T21:16:34,831 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/b6c0c3f31033450798a310a8beaa7bf7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/b6c0c3f31033450798a310a8beaa7bf7 2024-12-08T21:16:34,835 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/C of c482ca018e010a4a9e75b04b5a867252 into b6c0c3f31033450798a310a8beaa7bf7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:34,835 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:34,835 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/C, priority=12, startTime=1733692594791; duration=0sec 2024-12-08T21:16:34,835 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:34,835 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T21:16:34,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-08T21:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:34,931 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-08T21:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:34,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:34,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/9af4a60e3b554bb28985dc9aef28091d is 50, key is test_row_0/A:col10/1733692593552/Put/seqid=0 2024-12-08T21:16:34,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742311_1487 (size=12301) 2024-12-08T21:16:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:35,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. as already flushing 2024-12-08T21:16:35,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692595074, max timestamp=1733692595084Num uniques 859; 2024-12-08T21:16:35,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:16:35,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692595074, max timestamp=1733692595084Num uniques 859; 2024-12-08T21:16:35,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:16:35,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733692595074, max timestamp=1733692595084Num uniques 859; 2024-12-08T21:16:35,086 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-08T21:16:35,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49918 deadline: 1733692655128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692595110, max timestamp=1733692595131Num uniques 1068; 2024-12-08T21:16:35,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-08T21:16:35,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692655128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692595110, max timestamp=1733692595131Num uniques 1068; 2024-12-08T21:16:35,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692595110, max timestamp=1733692595131Num uniques 1068; 2024-12-08T21:16:35,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-08T21:16:35,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-08T21:16:35,134 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692655204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692595190, max timestamp=1733692595211Num uniques 1077; 2024-12-08T21:16:35,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-08T21:16:35,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692595190, max timestamp=1733692595211Num uniques 1077; 2024-12-08T21:16:35,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-08T21:16:35,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733692595190, max timestamp=1733692595211Num uniques 1077; 2024-12-08T21:16:35,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-08T21:16:35,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692595266, max timestamp=1733692595276Num uniques 1036; 2024-12-08T21:16:35,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-08T21:16:35,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692595266, max timestamp=1733692595276Num uniques 1036; 2024-12-08T21:16:35,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733692595266, max timestamp=1733692595276Num uniques 1036; 2024-12-08T21:16:35,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-08T21:16:35,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-08T21:16:35,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692595313, max timestamp=1733692595333Num uniques 1095; 2024-12-08T21:16:35,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-08T21:16:35,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692595313, max timestamp=1733692595333Num uniques 1095; 2024-12-08T21:16:35,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-08T21:16:35,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1733692595313, max timestamp=1733692595333Num uniques 1095; 2024-12-08T21:16:35,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-12-08T21:16:35,338 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/9af4a60e3b554bb28985dc9aef28091d 2024-12-08T21:16:35,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/396351d0cf594cbfa53d50cc8d7f633f is 50, key is test_row_0/B:col10/1733692593552/Put/seqid=0 2024-12-08T21:16:35,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742312_1488 (size=12301) 2024-12-08T21:16:35,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692655403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692655403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692655403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,416 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692655517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692655518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692655518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692655727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692655728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692655728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49932 deadline: 1733692655738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:35,747 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in c482ca018e010a4a9e75b04b5a867252 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:35,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:35,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:35,753 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/396351d0cf594cbfa53d50cc8d7f633f 2024-12-08T21:16:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a04f971ae07742d88742e38c5eb1129d is 50, key is test_row_0/C:col10/1733692593552/Put/seqid=0 2024-12-08T21:16:35,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742313_1489 (size=12301) 2024-12-08T21:16:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T21:16:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692656048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692656048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692656048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,182 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a04f971ae07742d88742e38c5eb1129d 2024-12-08T21:16:36,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/9af4a60e3b554bb28985dc9aef28091d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9af4a60e3b554bb28985dc9aef28091d 2024-12-08T21:16:36,189 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9af4a60e3b554bb28985dc9aef28091d, entries=150, sequenceid=388, filesize=12.0 K 2024-12-08T21:16:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/396351d0cf594cbfa53d50cc8d7f633f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/396351d0cf594cbfa53d50cc8d7f633f 2024-12-08T21:16:36,193 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/396351d0cf594cbfa53d50cc8d7f633f, entries=150, sequenceid=388, filesize=12.0 K 2024-12-08T21:16:36,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/a04f971ae07742d88742e38c5eb1129d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a04f971ae07742d88742e38c5eb1129d 2024-12-08T21:16:36,198 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a04f971ae07742d88742e38c5eb1129d, entries=150, sequenceid=388, filesize=12.0 K 2024-12-08T21:16:36,199 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c482ca018e010a4a9e75b04b5a867252 in 1268ms, sequenceid=388, compaction requested=false 2024-12-08T21:16:36,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:36,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:36,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-08T21:16:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-08T21:16:36,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-08T21:16:36,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4900 sec 2024-12-08T21:16:36,203 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.4940 sec 2024-12-08T21:16:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:36,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-12-08T21:16:36,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:36,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:36,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:36,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:36,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:36,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-08T21:16:36,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d2ac9da123e84f6f9f56d954d3bf957e is 50, key is test_row_0/A:col10/1733692595350/Put/seqid=0 2024-12-08T21:16:36,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742314_1490 (size=17181) 2024-12-08T21:16:36,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692656566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692656567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692656576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692656678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692656678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692656687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692656888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692656888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692656897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:36,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:36,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:36,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d2ac9da123e84f6f9f56d954d3bf957e 2024-12-08T21:16:36,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/8cb76bc724174219ace944f352b366a1 is 50, key is test_row_0/B:col10/1733692595350/Put/seqid=0 2024-12-08T21:16:36,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742315_1491 (size=12301) 2024-12-08T21:16:37,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49982 deadline: 1733692657197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:37,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49958 deadline: 1733692657197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:37,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:37,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49916 deadline: 1733692657208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., hostname=e653e208f502,38457,1733692444075, seqNum=2 because the exception is null or not the one we care about 2024-12-08T21:16:37,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/8cb76bc724174219ace944f352b366a1 2024-12-08T21:16:37,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/0e3d413fe6094fd79dc116afa9914326 is 50, key is test_row_0/C:col10/1733692595350/Put/seqid=0 2024-12-08T21:16:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742316_1492 (size=12301) 2024-12-08T21:16:37,391 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/0e3d413fe6094fd79dc116afa9914326 2024-12-08T21:16:37,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d2ac9da123e84f6f9f56d954d3bf957e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d2ac9da123e84f6f9f56d954d3bf957e 2024-12-08T21:16:37,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d2ac9da123e84f6f9f56d954d3bf957e, entries=250, sequenceid=496, filesize=16.8 K 2024-12-08T21:16:37,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/8cb76bc724174219ace944f352b366a1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/8cb76bc724174219ace944f352b366a1 2024-12-08T21:16:37,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/8cb76bc724174219ace944f352b366a1, entries=150, sequenceid=496, filesize=12.0 K 2024-12-08T21:16:37,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/0e3d413fe6094fd79dc116afa9914326 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0e3d413fe6094fd79dc116afa9914326 2024-12-08T21:16:37,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0e3d413fe6094fd79dc116afa9914326, entries=150, sequenceid=496, filesize=12.0 K 2024-12-08T21:16:37,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~457.27 KB/468240, currentSize=20.13 KB/20610 for c482ca018e010a4a9e75b04b5a867252 in 841ms, sequenceid=496, compaction requested=true 2024-12-08T21:16:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:37,405 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:37,405 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:37,406 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:37,406 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:37,406 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/B is initiating minor compaction (all files) 2024-12-08T21:16:37,406 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/A is initiating minor compaction (all files) 2024-12-08T21:16:37,406 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/B in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:37,406 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/A in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:37,406 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/bcc2a64bd4654f319f047c6df092cddf, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/396351d0cf594cbfa53d50cc8d7f633f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/8cb76bc724174219ace944f352b366a1] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=36.8 K 2024-12-08T21:16:37,406 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7a28660ec94448d8a04ee91f2977bd3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9af4a60e3b554bb28985dc9aef28091d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d2ac9da123e84f6f9f56d954d3bf957e] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=41.5 K 2024-12-08T21:16:37,407 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc2a64bd4654f319f047c6df092cddf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733692592882 2024-12-08T21:16:37,407 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7a28660ec94448d8a04ee91f2977bd3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733692592882 2024-12-08T21:16:37,407 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 396351d0cf594cbfa53d50cc8d7f633f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733692593552 2024-12-08T21:16:37,407 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af4a60e3b554bb28985dc9aef28091d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733692593552 2024-12-08T21:16:37,407 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb76bc724174219ace944f352b366a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733692595350 2024-12-08T21:16:37,407 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2ac9da123e84f6f9f56d954d3bf957e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733692595350 2024-12-08T21:16:37,413 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:37,413 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/0b52c74d6e204d4597dbcb6c97520e00 is 50, key is test_row_0/B:col10/1733692595350/Put/seqid=0 2024-12-08T21:16:37,416 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:37,416 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/83236c0c61ee4d51ab2132517800dace is 50, key is test_row_0/A:col10/1733692595350/Put/seqid=0 2024-12-08T21:16:37,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742317_1493 (size=13153) 2024-12-08T21:16:37,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742318_1494 (size=13153) 2024-12-08T21:16:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:37,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:16:37,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:37,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:37,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:37,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:37,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:37,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:37,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/eaca8c704bed41aaa45c8dd86970bdcb is 50, key is test_row_0/A:col10/1733692597719/Put/seqid=0 2024-12-08T21:16:37,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742319_1495 (size=14741) 2024-12-08T21:16:37,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692597732, max timestamp=1733692597761Num uniques 859; 2024-12-08T21:16:37,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692597732, max timestamp=1733692597761Num uniques 859; 2024-12-08T21:16:37,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:16:37,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:16:37,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692597732, max timestamp=1733692597761Num uniques 859; 2024-12-08T21:16:37,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:16:37,779 INFO [Thread-1908 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1908. 2024-12-08T21:16:37,779 DEBUG [Thread-1908 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,779 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,779 INFO [Thread-1891 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1891. 2024-12-08T21:16:37,779 DEBUG [Thread-1908 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,779 DEBUG [Thread-1891 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,779 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,780 DEBUG [Thread-1891 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,787 INFO [Thread-1906 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1906. 2024-12-08T21:16:37,787 DEBUG [Thread-1906 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,787 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,787 DEBUG [Thread-1906 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,787 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,787 INFO [Thread-1889 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1889. 2024-12-08T21:16:37,788 DEBUG [Thread-1889 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,788 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,788 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,788 INFO [Thread-1904 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1904. 2024-12-08T21:16:37,788 DEBUG [Thread-1904 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,788 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,788 DEBUG [Thread-1904 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,788 INFO [Thread-1893 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1893. 2024-12-08T21:16:37,788 DEBUG [Thread-1893 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,788 DEBUG [Thread-1893 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,789 DEBUG [Thread-1893 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,790 INFO [Thread-1895 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1895. 2024-12-08T21:16:37,790 DEBUG [Thread-1895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,790 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,790 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,791 INFO [Thread-1902 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1902. 2024-12-08T21:16:37,791 DEBUG [Thread-1902 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,791 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,791 DEBUG [Thread-1902 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,796 INFO [Thread-1900 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1900. 2024-12-08T21:16:37,796 DEBUG [Thread-1900 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:37,796 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,797 DEBUG [Thread-1900 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:37,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:37,822 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/0b52c74d6e204d4597dbcb6c97520e00 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/0b52c74d6e204d4597dbcb6c97520e00 2024-12-08T21:16:37,830 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/83236c0c61ee4d51ab2132517800dace as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/83236c0c61ee4d51ab2132517800dace 2024-12-08T21:16:37,832 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/B of c482ca018e010a4a9e75b04b5a867252 into 0b52c74d6e204d4597dbcb6c97520e00(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:37,832 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:37,832 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/B, priority=13, startTime=1733692597405; duration=0sec 2024-12-08T21:16:37,833 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:37,833 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:37,833 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:37,836 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:37,836 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): c482ca018e010a4a9e75b04b5a867252/C is initiating minor compaction (all files) 2024-12-08T21:16:37,836 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c482ca018e010a4a9e75b04b5a867252/C in TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:37,836 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/b6c0c3f31033450798a310a8beaa7bf7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a04f971ae07742d88742e38c5eb1129d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0e3d413fe6094fd79dc116afa9914326] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp, totalSize=36.8 K 2024-12-08T21:16:37,836 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c0c3f31033450798a310a8beaa7bf7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733692592882 2024-12-08T21:16:37,837 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a04f971ae07742d88742e38c5eb1129d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733692593552 2024-12-08T21:16:37,837 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e3d413fe6094fd79dc116afa9914326, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1733692595350 2024-12-08T21:16:37,838 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/A of c482ca018e010a4a9e75b04b5a867252 into 83236c0c61ee4d51ab2132517800dace(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:37,838 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:37,838 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/A, priority=13, startTime=1733692597405; duration=0sec 2024-12-08T21:16:37,839 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:37,839 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:37,843 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c482ca018e010a4a9e75b04b5a867252#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:37,844 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/27ba9c0d2f0d4599b1d6523ee1c543cb is 50, key is test_row_0/C:col10/1733692595350/Put/seqid=0 2024-12-08T21:16:37,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742320_1496 (size=13153) 2024-12-08T21:16:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-08T21:16:37,878 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:38,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/eaca8c704bed41aaa45c8dd86970bdcb 2024-12-08T21:16:38,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/4ae10c3d43d34e1c8ba9be465da8aff3 is 50, key is test_row_0/B:col10/1733692597719/Put/seqid=0 2024-12-08T21:16:38,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742321_1497 (size=12301) 2024-12-08T21:16:38,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/4ae10c3d43d34e1c8ba9be465da8aff3 2024-12-08T21:16:38,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/91bed38d7ca849108b40d02204eab089 is 50, key is test_row_0/C:col10/1733692597719/Put/seqid=0 2024-12-08T21:16:38,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742322_1498 (size=12301) 2024-12-08T21:16:38,257 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/27ba9c0d2f0d4599b1d6523ee1c543cb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/27ba9c0d2f0d4599b1d6523ee1c543cb 2024-12-08T21:16:38,261 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c482ca018e010a4a9e75b04b5a867252/C of c482ca018e010a4a9e75b04b5a867252 into 27ba9c0d2f0d4599b1d6523ee1c543cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:38,261 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:38,261 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252., storeName=c482ca018e010a4a9e75b04b5a867252/C, priority=13, startTime=1733692597405; duration=0sec 2024-12-08T21:16:38,261 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:38,261 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:38,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/91bed38d7ca849108b40d02204eab089 2024-12-08T21:16:38,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/eaca8c704bed41aaa45c8dd86970bdcb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/eaca8c704bed41aaa45c8dd86970bdcb 2024-12-08T21:16:38,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/eaca8c704bed41aaa45c8dd86970bdcb, entries=200, sequenceid=506, filesize=14.4 K 2024-12-08T21:16:38,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/4ae10c3d43d34e1c8ba9be465da8aff3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/4ae10c3d43d34e1c8ba9be465da8aff3 2024-12-08T21:16:38,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/4ae10c3d43d34e1c8ba9be465da8aff3, entries=150, sequenceid=506, filesize=12.0 K 2024-12-08T21:16:38,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/91bed38d7ca849108b40d02204eab089 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/91bed38d7ca849108b40d02204eab089 2024-12-08T21:16:38,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/91bed38d7ca849108b40d02204eab089, entries=150, sequenceid=506, filesize=12.0 K 2024-12-08T21:16:38,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=93.93 KB/96180 for c482ca018e010a4a9e75b04b5a867252 in 846ms, sequenceid=506, compaction requested=false 2024-12-08T21:16:38,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:39,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c482ca018e010a4a9e75b04b5a867252 3/3 column families, dataSize=100.63 KB heapSize=276 KB 2024-12-08T21:16:39,160 INFO [Thread-1897 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1897. 2024-12-08T21:16:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=A 2024-12-08T21:16:39,160 DEBUG [Thread-1897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:16:39,160 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:39,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=B 2024-12-08T21:16:39,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:39,161 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:39,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c482ca018e010a4a9e75b04b5a867252, store=C 2024-12-08T21:16:39,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:39,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1637 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4909 rows 2024-12-08T21:16:39,161 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1635 2024-12-08T21:16:39,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4900 rows 2024-12-08T21:16:39,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1630 2024-12-08T21:16:39,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4886 rows 2024-12-08T21:16:39,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1628 2024-12-08T21:16:39,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4883 rows 2024-12-08T21:16:39,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1628 2024-12-08T21:16:39,162 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4882 rows 2024-12-08T21:16:39,162 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:16:39,162 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:16:39,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:39,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:39,163 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:16:39,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T21:16:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T21:16:39,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d1d3ea8e05ed4920a20ebd6e824565c5 is 50, key is test_row_0/A:col10/1733692597786/Put/seqid=0 2024-12-08T21:16:39,167 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692599167"}]},"ts":"1733692599167"} 2024-12-08T21:16:39,169 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T21:16:39,169 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T21:16:39,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:16:39,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, UNASSIGN}] 2024-12-08T21:16:39,173 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, UNASSIGN 2024-12-08T21:16:39,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c482ca018e010a4a9e75b04b5a867252, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:39,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, UNASSIGN because future has completed 2024-12-08T21:16:39,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742323_1499 (size=12301) 2024-12-08T21:16:39,175 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:16:39,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:16:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T21:16:39,327 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:39,327 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:16:39,328 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing c482ca018e010a4a9e75b04b5a867252, disabling compactions & flushes 2024-12-08T21:16:39,328 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T21:16:39,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d1d3ea8e05ed4920a20ebd6e824565c5 2024-12-08T21:16:39,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/16b92c7e4f1545a3b7ce270964a60711 is 50, key is test_row_0/B:col10/1733692597786/Put/seqid=0 2024-12-08T21:16:39,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742324_1500 (size=12301) 2024-12-08T21:16:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T21:16:39,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/16b92c7e4f1545a3b7ce270964a60711 2024-12-08T21:16:40,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/0cc173845a804f5abe45a0e2e424c76e is 50, key is test_row_0/C:col10/1733692597786/Put/seqid=0 2024-12-08T21:16:40,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742325_1501 (size=12301) 2024-12-08T21:16:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T21:16:40,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/0cc173845a804f5abe45a0e2e424c76e 2024-12-08T21:16:40,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/A/d1d3ea8e05ed4920a20ebd6e824565c5 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1d3ea8e05ed4920a20ebd6e824565c5 2024-12-08T21:16:40,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1d3ea8e05ed4920a20ebd6e824565c5, entries=150, sequenceid=539, filesize=12.0 K 2024-12-08T21:16:40,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/B/16b92c7e4f1545a3b7ce270964a60711 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/16b92c7e4f1545a3b7ce270964a60711 2024-12-08T21:16:40,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/16b92c7e4f1545a3b7ce270964a60711, entries=150, sequenceid=539, filesize=12.0 K 2024-12-08T21:16:40,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/.tmp/C/0cc173845a804f5abe45a0e2e424c76e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0cc173845a804f5abe45a0e2e424c76e 2024-12-08T21:16:40,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0cc173845a804f5abe45a0e2e424c76e, entries=150, sequenceid=539, filesize=12.0 K 2024-12-08T21:16:40,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~275.95 KB/282576, currentSize=0 B/0 for c482ca018e010a4a9e75b04b5a867252 in 1271ms, sequenceid=539, compaction requested=true 2024-12-08T21:16:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c482ca018e010a4a9e75b04b5a867252: 2024-12-08T21:16:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:40,431 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:40,431 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. because compaction request was cancelled 2024-12-08T21:16:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:40,431 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:40,431 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. after waiting 0 ms 2024-12-08T21:16:40,431 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:A 2024-12-08T21:16:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:40,431 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:40,431 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. because compaction request was cancelled 2024-12-08T21:16:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c482ca018e010a4a9e75b04b5a867252:C, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:40,431 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:B 2024-12-08T21:16:40,431 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. because compaction request was cancelled 2024-12-08T21:16:40,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:40,431 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c482ca018e010a4a9e75b04b5a867252:C 2024-12-08T21:16:40,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9c165f30f9414fd18a0e123444534b49, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/35cfb1beab4e40ab89f1d98794adea46, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ac073ff542344d739a78d5f535b46179, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ffbfd992dbba4d66a25352074a9ad645, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/02feb66403f44babbdfa40549c79c9fd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c31e70a624f7429bad1b852685cd0c66, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/804ce5aa3a134dc682e2c7041abf8b67, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c19dcb5feef94e33b1c496e9bcc1ce46, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/7e3a9158a7cc44df841518939ea7e94b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d0fcdb50349d49b5aa2a2c967fdf5941, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7b1534d45c94d89957a66bbf2e26a06, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/46d6247385204365a164a11c49b3629e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a28f62660daa465ca2e8cdd24a9efcdd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/953cb3449a60410dbb2205eee5503058, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/0f196adb60c446d9816d1b9f5e943325, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f62de546ce7b449b8e41481d8d9444b5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a558d5ab260444b09d7531c2d7480452, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/6553fd82cb02420b992bf8d970ce0b10, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/61d4a1f5222e4a4a8503943c1b54c9dd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/381f16352c474732b6e763a1fe1589f8, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/10a1dd8aba2c4368b4aeb0aec79bbd7f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1deceb825474159975e9512808c8d50, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7a28660ec94448d8a04ee91f2977bd3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9af4a60e3b554bb28985dc9aef28091d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d2ac9da123e84f6f9f56d954d3bf957e] to archive 2024-12-08T21:16:40,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:16:40,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9c165f30f9414fd18a0e123444534b49 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9c165f30f9414fd18a0e123444534b49 2024-12-08T21:16:40,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/35cfb1beab4e40ab89f1d98794adea46 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/35cfb1beab4e40ab89f1d98794adea46 2024-12-08T21:16:40,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ac073ff542344d739a78d5f535b46179 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ac073ff542344d739a78d5f535b46179 2024-12-08T21:16:40,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ffbfd992dbba4d66a25352074a9ad645 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/ffbfd992dbba4d66a25352074a9ad645 2024-12-08T21:16:40,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/02feb66403f44babbdfa40549c79c9fd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/02feb66403f44babbdfa40549c79c9fd 2024-12-08T21:16:40,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c31e70a624f7429bad1b852685cd0c66 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c31e70a624f7429bad1b852685cd0c66 2024-12-08T21:16:40,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/804ce5aa3a134dc682e2c7041abf8b67 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/804ce5aa3a134dc682e2c7041abf8b67 2024-12-08T21:16:40,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c19dcb5feef94e33b1c496e9bcc1ce46 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/c19dcb5feef94e33b1c496e9bcc1ce46 2024-12-08T21:16:40,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/7e3a9158a7cc44df841518939ea7e94b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/7e3a9158a7cc44df841518939ea7e94b 2024-12-08T21:16:40,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d0fcdb50349d49b5aa2a2c967fdf5941 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d0fcdb50349d49b5aa2a2c967fdf5941 2024-12-08T21:16:40,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7b1534d45c94d89957a66bbf2e26a06 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7b1534d45c94d89957a66bbf2e26a06 2024-12-08T21:16:40,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/46d6247385204365a164a11c49b3629e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/46d6247385204365a164a11c49b3629e 2024-12-08T21:16:40,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a28f62660daa465ca2e8cdd24a9efcdd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a28f62660daa465ca2e8cdd24a9efcdd 2024-12-08T21:16:40,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/953cb3449a60410dbb2205eee5503058 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/953cb3449a60410dbb2205eee5503058 2024-12-08T21:16:40,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/0f196adb60c446d9816d1b9f5e943325 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/0f196adb60c446d9816d1b9f5e943325 2024-12-08T21:16:40,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f62de546ce7b449b8e41481d8d9444b5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f62de546ce7b449b8e41481d8d9444b5 2024-12-08T21:16:40,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a558d5ab260444b09d7531c2d7480452 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/a558d5ab260444b09d7531c2d7480452 2024-12-08T21:16:40,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/6553fd82cb02420b992bf8d970ce0b10 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/6553fd82cb02420b992bf8d970ce0b10 2024-12-08T21:16:40,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/61d4a1f5222e4a4a8503943c1b54c9dd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/61d4a1f5222e4a4a8503943c1b54c9dd 2024-12-08T21:16:40,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/381f16352c474732b6e763a1fe1589f8 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/381f16352c474732b6e763a1fe1589f8 2024-12-08T21:16:40,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/10a1dd8aba2c4368b4aeb0aec79bbd7f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/10a1dd8aba2c4368b4aeb0aec79bbd7f 2024-12-08T21:16:40,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1deceb825474159975e9512808c8d50 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1deceb825474159975e9512808c8d50 2024-12-08T21:16:40,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7a28660ec94448d8a04ee91f2977bd3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/f7a28660ec94448d8a04ee91f2977bd3 2024-12-08T21:16:40,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9af4a60e3b554bb28985dc9aef28091d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/9af4a60e3b554bb28985dc9aef28091d 2024-12-08T21:16:40,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d2ac9da123e84f6f9f56d954d3bf957e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d2ac9da123e84f6f9f56d954d3bf957e 2024-12-08T21:16:40,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/ae69b99a92ac4731a1591bcd38c15b4e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/cec4c1099eb84586a5d3936115f25d05, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b248195fff074e329e5b90380b6881b0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/1777fd4e110446d9a209f005d084d121, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/169e84455c774f10ba712229f49c12e0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/148946406a6349c9b63b6db80d9da210, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e87c8f7a060a4208a961c757bfe3ee5f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/afcab021f29249ad9a65b4c52209c4dd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/6c2daead9e064635bcd572fbe2cf08b7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/37989a3b842d4ad8bcaddd31f57959db, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e3b5c04f119f406e9d90a75684c811d0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/3afd39c7c68b43959e12e69256a2e55e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/c8b60721dfa9490cae1c35c97e1edb5f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b5a74149da9047d0b468f69f841c5658, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/acae0d5c1e364be6b5b963b8f6dd14fd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a6be96e81b1442bf9361f180fed95723, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/144d6dbc45bf47f9adb85aa7a092090a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/008057e315fc497aa35629fbed5a9586, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/f7427b01cc5d42aa864ca410e37dfb67, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a3dcb80f364240db9feb03676a1ab78f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/26b59fc648af45e09623a64294eece05, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/bcc2a64bd4654f319f047c6df092cddf, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/df5c57aad3074ef899539950ad4a7e25, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/396351d0cf594cbfa53d50cc8d7f633f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/8cb76bc724174219ace944f352b366a1] to archive 2024-12-08T21:16:40,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:16:40,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/ae69b99a92ac4731a1591bcd38c15b4e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/ae69b99a92ac4731a1591bcd38c15b4e 2024-12-08T21:16:40,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/cec4c1099eb84586a5d3936115f25d05 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/cec4c1099eb84586a5d3936115f25d05 2024-12-08T21:16:40,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b248195fff074e329e5b90380b6881b0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b248195fff074e329e5b90380b6881b0 2024-12-08T21:16:40,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/1777fd4e110446d9a209f005d084d121 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/1777fd4e110446d9a209f005d084d121 2024-12-08T21:16:40,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/169e84455c774f10ba712229f49c12e0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/169e84455c774f10ba712229f49c12e0 2024-12-08T21:16:40,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/148946406a6349c9b63b6db80d9da210 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/148946406a6349c9b63b6db80d9da210 2024-12-08T21:16:40,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e87c8f7a060a4208a961c757bfe3ee5f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e87c8f7a060a4208a961c757bfe3ee5f 2024-12-08T21:16:40,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/afcab021f29249ad9a65b4c52209c4dd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/afcab021f29249ad9a65b4c52209c4dd 2024-12-08T21:16:40,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/6c2daead9e064635bcd572fbe2cf08b7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/6c2daead9e064635bcd572fbe2cf08b7 2024-12-08T21:16:40,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/37989a3b842d4ad8bcaddd31f57959db to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/37989a3b842d4ad8bcaddd31f57959db 2024-12-08T21:16:40,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e3b5c04f119f406e9d90a75684c811d0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/e3b5c04f119f406e9d90a75684c811d0 2024-12-08T21:16:40,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/3afd39c7c68b43959e12e69256a2e55e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/3afd39c7c68b43959e12e69256a2e55e 2024-12-08T21:16:40,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/c8b60721dfa9490cae1c35c97e1edb5f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/c8b60721dfa9490cae1c35c97e1edb5f 2024-12-08T21:16:40,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b5a74149da9047d0b468f69f841c5658 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/b5a74149da9047d0b468f69f841c5658 2024-12-08T21:16:40,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/acae0d5c1e364be6b5b963b8f6dd14fd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/acae0d5c1e364be6b5b963b8f6dd14fd 2024-12-08T21:16:40,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a6be96e81b1442bf9361f180fed95723 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a6be96e81b1442bf9361f180fed95723 2024-12-08T21:16:40,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/144d6dbc45bf47f9adb85aa7a092090a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/144d6dbc45bf47f9adb85aa7a092090a 2024-12-08T21:16:40,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/008057e315fc497aa35629fbed5a9586 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/008057e315fc497aa35629fbed5a9586 2024-12-08T21:16:40,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/f7427b01cc5d42aa864ca410e37dfb67 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/f7427b01cc5d42aa864ca410e37dfb67 2024-12-08T21:16:40,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a3dcb80f364240db9feb03676a1ab78f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/a3dcb80f364240db9feb03676a1ab78f 2024-12-08T21:16:40,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/26b59fc648af45e09623a64294eece05 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/26b59fc648af45e09623a64294eece05 2024-12-08T21:16:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/bcc2a64bd4654f319f047c6df092cddf to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/bcc2a64bd4654f319f047c6df092cddf 2024-12-08T21:16:40,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/df5c57aad3074ef899539950ad4a7e25 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/df5c57aad3074ef899539950ad4a7e25 2024-12-08T21:16:40,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/396351d0cf594cbfa53d50cc8d7f633f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/396351d0cf594cbfa53d50cc8d7f633f 2024-12-08T21:16:40,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/8cb76bc724174219ace944f352b366a1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/8cb76bc724174219ace944f352b366a1 2024-12-08T21:16:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8316f537b205443fa9309d145d96b6e9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f44d0b61c59740c785fbd511f4cf495a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/23bea873a4da4c88b077c3efe8346f82, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/5a061befdf2047b8883e2f021598a9f5, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/cc2d6d5142de4475a91ce592ec6f58c7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8d5e79388e5e4c998ec85465337df52b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/054b50d6de904c4ba82fbc3d1178bff7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f0300180a7744835a475189342bfdbb0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/4294363ab102467d94866a9eb16c755f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a5f483d20efb42a69894daa17e0cd474, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/208e443339fc40f7add324dacc96e15a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/00f4336447a94e749b6fc98ff39e3963, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f4b06b0e5aaa41308606b2daeed5cb1d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/fcf6d735fb0e4abaa01b34c030a7f43d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/24360375fa7a441f90ee7c019f406169, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a3c1bda7ff0b4e138f9e7fef71e0757e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/2759280924f74bf4a45fa5e838569723, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/024978f63e3141d78712cb9ea1b8e932, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/62c6df2906594de282b6d514b99f976a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/78e45901c82d49059f0f7fb2d01a7e2e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a6c802d6390340dea2e92fdd6eb6c6ae, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/b6c0c3f31033450798a310a8beaa7bf7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f1e5bdcc9039483b8a549f5c9956d292, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a04f971ae07742d88742e38c5eb1129d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0e3d413fe6094fd79dc116afa9914326] to archive 2024-12-08T21:16:40,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:16:40,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8316f537b205443fa9309d145d96b6e9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8316f537b205443fa9309d145d96b6e9 2024-12-08T21:16:40,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f44d0b61c59740c785fbd511f4cf495a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f44d0b61c59740c785fbd511f4cf495a 2024-12-08T21:16:40,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/23bea873a4da4c88b077c3efe8346f82 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/23bea873a4da4c88b077c3efe8346f82 2024-12-08T21:16:40,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/5a061befdf2047b8883e2f021598a9f5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/5a061befdf2047b8883e2f021598a9f5 2024-12-08T21:16:40,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/cc2d6d5142de4475a91ce592ec6f58c7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/cc2d6d5142de4475a91ce592ec6f58c7 2024-12-08T21:16:40,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8d5e79388e5e4c998ec85465337df52b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/8d5e79388e5e4c998ec85465337df52b 2024-12-08T21:16:40,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/054b50d6de904c4ba82fbc3d1178bff7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/054b50d6de904c4ba82fbc3d1178bff7 2024-12-08T21:16:40,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f0300180a7744835a475189342bfdbb0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f0300180a7744835a475189342bfdbb0 2024-12-08T21:16:40,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/4294363ab102467d94866a9eb16c755f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/4294363ab102467d94866a9eb16c755f 2024-12-08T21:16:40,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a5f483d20efb42a69894daa17e0cd474 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a5f483d20efb42a69894daa17e0cd474 2024-12-08T21:16:40,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/208e443339fc40f7add324dacc96e15a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/208e443339fc40f7add324dacc96e15a 2024-12-08T21:16:40,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/00f4336447a94e749b6fc98ff39e3963 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/00f4336447a94e749b6fc98ff39e3963 2024-12-08T21:16:40,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f4b06b0e5aaa41308606b2daeed5cb1d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f4b06b0e5aaa41308606b2daeed5cb1d 2024-12-08T21:16:40,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/fcf6d735fb0e4abaa01b34c030a7f43d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/fcf6d735fb0e4abaa01b34c030a7f43d 2024-12-08T21:16:40,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/24360375fa7a441f90ee7c019f406169 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/24360375fa7a441f90ee7c019f406169 2024-12-08T21:16:40,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a3c1bda7ff0b4e138f9e7fef71e0757e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a3c1bda7ff0b4e138f9e7fef71e0757e 2024-12-08T21:16:40,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/2759280924f74bf4a45fa5e838569723 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/2759280924f74bf4a45fa5e838569723 2024-12-08T21:16:40,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/024978f63e3141d78712cb9ea1b8e932 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/024978f63e3141d78712cb9ea1b8e932 2024-12-08T21:16:40,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/62c6df2906594de282b6d514b99f976a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/62c6df2906594de282b6d514b99f976a 2024-12-08T21:16:40,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/78e45901c82d49059f0f7fb2d01a7e2e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/78e45901c82d49059f0f7fb2d01a7e2e 2024-12-08T21:16:40,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a6c802d6390340dea2e92fdd6eb6c6ae to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a6c802d6390340dea2e92fdd6eb6c6ae 2024-12-08T21:16:40,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/b6c0c3f31033450798a310a8beaa7bf7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/b6c0c3f31033450798a310a8beaa7bf7 2024-12-08T21:16:40,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f1e5bdcc9039483b8a549f5c9956d292 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/f1e5bdcc9039483b8a549f5c9956d292 2024-12-08T21:16:40,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a04f971ae07742d88742e38c5eb1129d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/a04f971ae07742d88742e38c5eb1129d 2024-12-08T21:16:40,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0e3d413fe6094fd79dc116afa9914326 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0e3d413fe6094fd79dc116afa9914326 2024-12-08T21:16:40,503 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/recovered.edits/542.seqid, newMaxSeqId=542, maxSeqId=1 2024-12-08T21:16:40,504 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252. 2024-12-08T21:16:40,504 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for c482ca018e010a4a9e75b04b5a867252: Waiting for close lock at 1733692599328Running coprocessor pre-close hooks at 1733692599328Disabling compacts and flushes for region at 1733692599328Disabling writes for close at 1733692600431 (+1103 ms)Writing region close event to WAL at 1733692600501 (+70 ms)Running coprocessor post-close hooks at 1733692600504 (+3 ms)Closed at 1733692600504 2024-12-08T21:16:40,506 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:40,506 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c482ca018e010a4a9e75b04b5a867252, regionState=CLOSED 2024-12-08T21:16:40,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:16:40,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-08T21:16:40,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure c482ca018e010a4a9e75b04b5a867252, server=e653e208f502,38457,1733692444075 in 1.3330 sec 2024-12-08T21:16:40,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-12-08T21:16:40,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c482ca018e010a4a9e75b04b5a867252, UNASSIGN in 1.3370 sec 2024-12-08T21:16:40,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-08T21:16:40,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3410 sec 2024-12-08T21:16:40,513 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692600513"}]},"ts":"1733692600513"} 2024-12-08T21:16:40,514 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T21:16:40,514 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T21:16:40,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3510 sec 2024-12-08T21:16:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-08T21:16:41,320 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:41,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T21:16:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:41,326 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T21:16:41,328 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:41,331 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:41,333 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/recovered.edits] 2024-12-08T21:16:41,336 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/83236c0c61ee4d51ab2132517800dace to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/83236c0c61ee4d51ab2132517800dace 2024-12-08T21:16:41,337 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1d3ea8e05ed4920a20ebd6e824565c5 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/d1d3ea8e05ed4920a20ebd6e824565c5 2024-12-08T21:16:41,338 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/eaca8c704bed41aaa45c8dd86970bdcb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/A/eaca8c704bed41aaa45c8dd86970bdcb 2024-12-08T21:16:41,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/0b52c74d6e204d4597dbcb6c97520e00 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/0b52c74d6e204d4597dbcb6c97520e00 2024-12-08T21:16:41,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/16b92c7e4f1545a3b7ce270964a60711 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/16b92c7e4f1545a3b7ce270964a60711 2024-12-08T21:16:41,341 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/4ae10c3d43d34e1c8ba9be465da8aff3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/B/4ae10c3d43d34e1c8ba9be465da8aff3 2024-12-08T21:16:41,343 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0cc173845a804f5abe45a0e2e424c76e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/0cc173845a804f5abe45a0e2e424c76e 2024-12-08T21:16:41,343 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/27ba9c0d2f0d4599b1d6523ee1c543cb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/27ba9c0d2f0d4599b1d6523ee1c543cb 2024-12-08T21:16:41,344 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/91bed38d7ca849108b40d02204eab089 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/C/91bed38d7ca849108b40d02204eab089 2024-12-08T21:16:41,346 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/recovered.edits/542.seqid to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252/recovered.edits/542.seqid 2024-12-08T21:16:41,347 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/c482ca018e010a4a9e75b04b5a867252 2024-12-08T21:16:41,347 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T21:16:41,349 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:41,351 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T21:16:41,353 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T21:16:41,354 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:41,354 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T21:16:41,354 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733692601354"}]},"ts":"9223372036854775807"} 2024-12-08T21:16:41,356 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T21:16:41,356 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c482ca018e010a4a9e75b04b5a867252, NAME => 'TestAcidGuarantees,,1733692575579.c482ca018e010a4a9e75b04b5a867252.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T21:16:41,356 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T21:16:41,357 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733692601356"}]},"ts":"9223372036854775807"} 2024-12-08T21:16:41,359 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T21:16:41,359 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:41,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-12-08T21:16:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-08T21:16:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T21:16:41,439 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:41,454 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 248) - Thread LEAK? -, OpenFileDescriptor=447 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=551 (was 472) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8210 (was 8943) 2024-12-08T21:16:41,464 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=551, ProcessCount=11, AvailableMemoryMB=8210 2024-12-08T21:16:41,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:16:41,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:16:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:41,481 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-08T21:16:41,481 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:41,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-12-08T21:16:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-08T21:16:41,482 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-08T21:16:41,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742326_1502 (size=1061) 2024-12-08T21:16:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-08T21:16:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-08T21:16:41,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd 2024-12-08T21:16:41,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742327_1503 (size=53) 2024-12-08T21:16:42,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-08T21:16:42,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:16:42,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0bad5c3cd9edbcf653e271e4dc44fdec, disabling compactions & flushes 2024-12-08T21:16:42,307 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,307 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. after waiting 0 ms 2024-12-08T21:16:42,308 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,308 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,308 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0bad5c3cd9edbcf653e271e4dc44fdec: Waiting for close lock at 1733692602307Disabling compacts and flushes for region at 1733692602307Disabling writes for close at 1733692602308 (+1 ms)Writing region close event to WAL at 1733692602308Closed at 1733692602308 2024-12-08T21:16:42,311 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-08T21:16:42,311 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733692602311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733692602311"}]},"ts":"1733692602311"} 2024-12-08T21:16:42,315 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-08T21:16:42,317 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-08T21:16:42,317 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692602317"}]},"ts":"1733692602317"} 2024-12-08T21:16:42,318 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-08T21:16:42,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, ASSIGN}] 2024-12-08T21:16:42,319 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, ASSIGN 2024-12-08T21:16:42,320 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, ASSIGN; state=OFFLINE, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=false 2024-12-08T21:16:42,471 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:42,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, ASSIGN because future has completed 2024-12-08T21:16:42,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:16:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-08T21:16:42,633 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,633 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:16:42,634 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,634 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:16:42,635 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,635 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,637 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,639 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:42,639 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bad5c3cd9edbcf653e271e4dc44fdec columnFamilyName A 2024-12-08T21:16:42,639 DEBUG [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:42,640 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(327): Store=0bad5c3cd9edbcf653e271e4dc44fdec/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:42,640 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,642 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:42,642 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bad5c3cd9edbcf653e271e4dc44fdec columnFamilyName B 2024-12-08T21:16:42,642 DEBUG [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:42,643 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(327): Store=0bad5c3cd9edbcf653e271e4dc44fdec/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:42,643 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,644 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:42,644 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bad5c3cd9edbcf653e271e4dc44fdec columnFamilyName C 2024-12-08T21:16:42,644 DEBUG [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:42,645 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(327): Store=0bad5c3cd9edbcf653e271e4dc44fdec/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:42,645 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,645 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,646 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,647 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,648 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,648 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,649 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:16:42,651 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,654 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-08T21:16:42,654 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 0bad5c3cd9edbcf653e271e4dc44fdec; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71714216, jitterRate=0.06862509250640869}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:16:42,654 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:42,655 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 0bad5c3cd9edbcf653e271e4dc44fdec: Running coprocessor pre-open hook at 1733692602635Writing region info on filesystem at 1733692602635Initializing all the Stores at 1733692602637 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692602637Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692602637Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692602637Cleaning up temporary data from old regions at 1733692602648 (+11 ms)Running coprocessor post-open hooks at 1733692602654 (+6 ms)Region opened successfully at 1733692602655 (+1 ms) 2024-12-08T21:16:42,656 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., pid=138, masterSystemTime=1733692602626 2024-12-08T21:16:42,657 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,658 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:42,658 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=OPEN, openSeqNum=2, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:42,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:16:42,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-08T21:16:42,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 in 187 msec 2024-12-08T21:16:42,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-12-08T21:16:42,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, ASSIGN in 342 msec 2024-12-08T21:16:42,663 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-08T21:16:42,664 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692602663"}]},"ts":"1733692602663"} 2024-12-08T21:16:42,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-08T21:16:42,665 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-08T21:16:42,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2000 sec 2024-12-08T21:16:43,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-12-08T21:16:43,628 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:43,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:43,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:43,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:43,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9efcc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:43,634 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:43,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe5e4e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:43,636 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:43,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:43,639 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-08T21:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e653e208f502,45981,1733692443391 2024-12-08T21:16:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@140d993a 2024-12-08T21:16:43,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-08T21:16:43,642 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-08T21:16:43,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-08T21:16:43,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-08T21:16:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-08T21:16:43,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742328_1504 (size=1097) 2024-12-08T21:16:44,055 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-08T21:16:44,055 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-08T21:16:44,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:16:44,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, REOPEN/MOVE}] 2024-12-08T21:16:44,060 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, REOPEN/MOVE 2024-12-08T21:16:44,060 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, REOPEN/MOVE because future has completed 2024-12-08T21:16:44,061 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:16:44,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:16:44,214 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,214 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:16:44,214 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 0bad5c3cd9edbcf653e271e4dc44fdec, disabling compactions & flushes 2024-12-08T21:16:44,214 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,214 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,215 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. after waiting 0 ms 2024-12-08T21:16:44,215 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,222 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-08T21:16:44,223 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,223 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 0bad5c3cd9edbcf653e271e4dc44fdec: Waiting for close lock at 1733692604214Running coprocessor pre-close hooks at 1733692604214Disabling compacts and flushes for region at 1733692604214Disabling writes for close at 1733692604215 (+1 ms)Writing region close event to WAL at 1733692604215Running coprocessor post-close hooks at 1733692604223 (+8 ms)Closed at 1733692604223 2024-12-08T21:16:44,223 WARN [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 0bad5c3cd9edbcf653e271e4dc44fdec to self. 2024-12-08T21:16:44,226 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,227 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=CLOSED 2024-12-08T21:16:44,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:16:44,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-08T21:16:44,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 in 171 msec 2024-12-08T21:16:44,236 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, REOPEN/MOVE; state=CLOSED, location=e653e208f502,38457,1733692444075; forceNewPlan=false, retain=true 2024-12-08T21:16:44,388 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=OPENING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, REOPEN/MOVE because future has completed 2024-12-08T21:16:44,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:16:44,549 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,549 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} 2024-12-08T21:16:44,549 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,549 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-08T21:16:44,549 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,549 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,550 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,551 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:44,551 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bad5c3cd9edbcf653e271e4dc44fdec columnFamilyName A 2024-12-08T21:16:44,552 DEBUG [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:44,552 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(327): Store=0bad5c3cd9edbcf653e271e4dc44fdec/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:44,553 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,553 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:44,553 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bad5c3cd9edbcf653e271e4dc44fdec columnFamilyName B 2024-12-08T21:16:44,553 DEBUG [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:44,554 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(327): Store=0bad5c3cd9edbcf653e271e4dc44fdec/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:44,554 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,554 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-08T21:16:44,555 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bad5c3cd9edbcf653e271e4dc44fdec columnFamilyName C 2024-12-08T21:16:44,555 DEBUG [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:44,555 INFO [StoreOpener-0bad5c3cd9edbcf653e271e4dc44fdec-1 {}] regionserver.HStore(327): Store=0bad5c3cd9edbcf653e271e4dc44fdec/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-08T21:16:44,555 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,555 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,556 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,557 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,558 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,558 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,558 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-08T21:16:44,559 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,560 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 0bad5c3cd9edbcf653e271e4dc44fdec; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63043112, jitterRate=-0.060584425926208496}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-08T21:16:44,560 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,560 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 0bad5c3cd9edbcf653e271e4dc44fdec: Running coprocessor pre-open hook at 1733692604549Writing region info on filesystem at 1733692604549Initializing all the Stores at 1733692604550 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692604550Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692604550Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733692604550Cleaning up temporary data from old regions at 1733692604558 (+8 ms)Running coprocessor post-open hooks at 1733692604560 (+2 ms)Region opened successfully at 1733692604560 2024-12-08T21:16:44,561 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., pid=143, masterSystemTime=1733692604547 2024-12-08T21:16:44,562 DEBUG [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,563 INFO [RS_OPEN_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,563 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=OPEN, openSeqNum=5, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:16:44,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-12-08T21:16:44,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 in 173 msec 2024-12-08T21:16:44,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-08T21:16:44,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, REOPEN/MOVE in 507 msec 2024-12-08T21:16:44,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-08T21:16:44,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-12-08T21:16:44,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-12-08T21:16:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-08T21:16:44,573 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:44,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,575 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8af9a29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,576 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78753a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e448f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,578 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae721a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c73ff2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c4a011e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53953479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bad14b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f56855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2883834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,587 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df72af2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320fd5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e52681, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141a9871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ed7f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,594 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a484a73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,596 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab37b51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7026df0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e653e208f502,45981,-1 for getting cluster id 2024-12-08T21:16:44,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-08T21:16:44,599 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a8351f1-229c-4b50-b24b-8c67e222f303' 2024-12-08T21:16:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-08T21:16:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a8351f1-229c-4b50-b24b-8c67e222f303" 2024-12-08T21:16:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2cb2e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e653e208f502,45981,-1] 2024-12-08T21:16:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-08T21:16:44,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:16:44,600 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-08T21:16:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@504dfc9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-08T21:16:44,600 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,600 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,600 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,601 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,601 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,601 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,601 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,601 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,602 DEBUG [Thread-2269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,602 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-08T21:16:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,602 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e653e208f502,38457,1733692444075, seqNum=-1] 2024-12-08T21:16:44,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-08T21:16:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-12-08T21:16:44,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-08T21:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,605 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:44,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T21:16:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:44,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5] 2024-12-08T21:16:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:44,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:44,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:44,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120842f32fbb319a4d0288384e60071f9e0d_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692604610/Put/seqid=0 2024-12-08T21:16:44,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692604623, max timestamp=1733692604630Num uniques 859; 2024-12-08T21:16:44,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:44,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692604623, max timestamp=1733692604630Num uniques 859; 2024-12-08T21:16:44,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692604623, max timestamp=1733692604630Num uniques 859; 2024-12-08T21:16:44,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:44,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742329_1505 (size=12154) 2024-12-08T21:16:44,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692664644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692664644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692664645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692664646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692664647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T21:16:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692664758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692664758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692664758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692664758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692664758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:44,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:44,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:44,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:44,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:44,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:44,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T21:16:44,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692664968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692664968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692664968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692664968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692664968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:44,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,036 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:45,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120842f32fbb319a4d0288384e60071f9e0d_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120842f32fbb319a4d0288384e60071f9e0d_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:45,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/ea064dfd9c884f38a4ac9650c8ec34c9, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:45,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/ea064dfd9c884f38a4ac9650c8ec34c9 is 175, key is test_row_0/A:col10/1733692604610/Put/seqid=0 2024-12-08T21:16:45,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742330_1506 (size=30955) 2024-12-08T21:16:45,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:45,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:45,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T21:16:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692665277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692665278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692665278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692665278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692665278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:45,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:45,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/ea064dfd9c884f38a4ac9650c8ec34c9 2024-12-08T21:16:45,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/005b5bf541b44dd29e580af63ce21392 is 50, key is test_row_0/B:col10/1733692604610/Put/seqid=0 2024-12-08T21:16:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742331_1507 (size=12001) 2024-12-08T21:16:45,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:45,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:45,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:45,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:45,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T21:16:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692665788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692665788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692665788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692665788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692665788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:45,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:45,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/005b5bf541b44dd29e580af63ce21392 2024-12-08T21:16:45,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/e3d41a1833b94d0d93f22476310188c6 is 50, key is test_row_0/C:col10/1733692604610/Put/seqid=0 2024-12-08T21:16:45,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742332_1508 (size=12001) 2024-12-08T21:16:45,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/e3d41a1833b94d0d93f22476310188c6 2024-12-08T21:16:45,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/ea064dfd9c884f38a4ac9650c8ec34c9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/ea064dfd9c884f38a4ac9650c8ec34c9 2024-12-08T21:16:45,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/ea064dfd9c884f38a4ac9650c8ec34c9, entries=150, sequenceid=14, filesize=30.2 K 2024-12-08T21:16:45,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/005b5bf541b44dd29e580af63ce21392 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/005b5bf541b44dd29e580af63ce21392 2024-12-08T21:16:45,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/005b5bf541b44dd29e580af63ce21392, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T21:16:45,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/e3d41a1833b94d0d93f22476310188c6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/e3d41a1833b94d0d93f22476310188c6 2024-12-08T21:16:45,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/e3d41a1833b94d0d93f22476310188c6, entries=150, sequenceid=14, filesize=11.7 K 2024-12-08T21:16:45,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1297ms, sequenceid=14, compaction requested=false 2024-12-08T21:16:45,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:45,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-12-08T21:16:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:45,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T21:16:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:45,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe6d5fad22ec4f65ab31eda120c5f249_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692604646/Put/seqid=0 2024-12-08T21:16:45,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742333_1509 (size=12154) 2024-12-08T21:16:45,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:45,994 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208fe6d5fad22ec4f65ab31eda120c5f249_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe6d5fad22ec4f65ab31eda120c5f249_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:45,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/227700aeb7d0425082ea7554f5120481, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:45,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/227700aeb7d0425082ea7554f5120481 is 175, key is test_row_0/A:col10/1733692604646/Put/seqid=0 2024-12-08T21:16:45,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742334_1510 (size=30955) 2024-12-08T21:16:46,023 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-08T21:16:46,399 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/227700aeb7d0425082ea7554f5120481 2024-12-08T21:16:46,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/445204b19b92461fb3265d8f855951d0 is 50, key is test_row_0/B:col10/1733692604646/Put/seqid=0 2024-12-08T21:16:46,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742335_1511 (size=12001) 2024-12-08T21:16:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T21:16:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:46,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:46,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692666802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692666802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692666802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,808 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/445204b19b92461fb3265d8f855951d0 2024-12-08T21:16:46,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692666806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692666807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/c4ddc743aba24e458092394da6da338e is 50, key is test_row_0/C:col10/1733692604646/Put/seqid=0 2024-12-08T21:16:46,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742336_1512 (size=12001) 2024-12-08T21:16:46,818 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/c4ddc743aba24e458092394da6da338e 2024-12-08T21:16:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/227700aeb7d0425082ea7554f5120481 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/227700aeb7d0425082ea7554f5120481 2024-12-08T21:16:46,824 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/227700aeb7d0425082ea7554f5120481, entries=150, sequenceid=51, filesize=30.2 K 2024-12-08T21:16:46,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/445204b19b92461fb3265d8f855951d0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/445204b19b92461fb3265d8f855951d0 2024-12-08T21:16:46,827 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/445204b19b92461fb3265d8f855951d0, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T21:16:46,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/c4ddc743aba24e458092394da6da338e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c4ddc743aba24e458092394da6da338e 2024-12-08T21:16:46,832 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c4ddc743aba24e458092394da6da338e, entries=150, sequenceid=51, filesize=11.7 K 2024-12-08T21:16:46,832 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 0bad5c3cd9edbcf653e271e4dc44fdec in 850ms, sequenceid=51, compaction requested=false 2024-12-08T21:16:46,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:46,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:46,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-12-08T21:16:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-12-08T21:16:46,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-08T21:16:46,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2260 sec 2024-12-08T21:16:46,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.2340 sec 2024-12-08T21:16:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:46,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T21:16:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:46,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:46,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082f3be550ab29467ca47096482ad3b946_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692606801/Put/seqid=0 2024-12-08T21:16:46,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742337_1513 (size=14594) 2024-12-08T21:16:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692666933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692666934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692666935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692666936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692666937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692667048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692667048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692667048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692667048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692667048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692667258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692667258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692667258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692667258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692667258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:47,342 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082f3be550ab29467ca47096482ad3b946_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082f3be550ab29467ca47096482ad3b946_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:47,343 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3c057df96583479c9bb6ea54ff867596, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:47,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3c057df96583479c9bb6ea54ff867596 is 175, key is test_row_0/A:col10/1733692606801/Put/seqid=0 2024-12-08T21:16:47,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742338_1514 (size=39549) 2024-12-08T21:16:47,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3c057df96583479c9bb6ea54ff867596 2024-12-08T21:16:47,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b2449ae1495e4adf8f6952350a3e7d53 is 50, key is test_row_0/B:col10/1733692606801/Put/seqid=0 2024-12-08T21:16:47,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742339_1515 (size=12001) 2024-12-08T21:16:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692667568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692667568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692667568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692667568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692667568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:47,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:47,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b2449ae1495e4adf8f6952350a3e7d53 2024-12-08T21:16:47,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/9c460361a84f4467b620ee6897268737 is 50, key is test_row_0/C:col10/1733692606801/Put/seqid=0 2024-12-08T21:16:47,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742340_1516 (size=12001) 2024-12-08T21:16:47,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/9c460361a84f4467b620ee6897268737 2024-12-08T21:16:47,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3c057df96583479c9bb6ea54ff867596 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3c057df96583479c9bb6ea54ff867596 2024-12-08T21:16:47,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3c057df96583479c9bb6ea54ff867596, entries=200, sequenceid=63, filesize=38.6 K 2024-12-08T21:16:47,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b2449ae1495e4adf8f6952350a3e7d53 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b2449ae1495e4adf8f6952350a3e7d53 2024-12-08T21:16:47,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b2449ae1495e4adf8f6952350a3e7d53, entries=150, sequenceid=63, filesize=11.7 K 2024-12-08T21:16:47,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/9c460361a84f4467b620ee6897268737 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9c460361a84f4467b620ee6897268737 2024-12-08T21:16:47,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9c460361a84f4467b620ee6897268737, entries=150, sequenceid=63, filesize=11.7 K 2024-12-08T21:16:47,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0bad5c3cd9edbcf653e271e4dc44fdec in 890ms, sequenceid=63, compaction requested=true 2024-12-08T21:16:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:47,809 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:47,809 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:16:47,810 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:47,810 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:47,810 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/ea064dfd9c884f38a4ac9650c8ec34c9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/227700aeb7d0425082ea7554f5120481, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3c057df96583479c9bb6ea54ff867596] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=99.1 K 2024-12-08T21:16:47,810 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/005b5bf541b44dd29e580af63ce21392, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/445204b19b92461fb3265d8f855951d0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b2449ae1495e4adf8f6952350a3e7d53] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.2 K 2024-12-08T21:16:47,810 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/ea064dfd9c884f38a4ac9650c8ec34c9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/227700aeb7d0425082ea7554f5120481, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3c057df96583479c9bb6ea54ff867596] 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 005b5bf541b44dd29e580af63ce21392, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733692604608 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea064dfd9c884f38a4ac9650c8ec34c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733692604608 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 445204b19b92461fb3265d8f855951d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733692604644 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 227700aeb7d0425082ea7554f5120481, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733692604644 2024-12-08T21:16:47,810 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b2449ae1495e4adf8f6952350a3e7d53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733692606801 2024-12-08T21:16:47,811 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c057df96583479c9bb6ea54ff867596, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733692606801 2024-12-08T21:16:47,825 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:47,825 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/962853e7b2b94402b612c09dd4819000 is 50, key is test_row_0/B:col10/1733692606801/Put/seqid=0 2024-12-08T21:16:47,826 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:47,830 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208f8dd27af9c994d4c9f75db1f49e73ad9_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:47,831 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208f8dd27af9c994d4c9f75db1f49e73ad9_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:47,831 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208f8dd27af9c994d4c9f75db1f49e73ad9_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:47,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742341_1517 (size=12104) 2024-12-08T21:16:47,841 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/962853e7b2b94402b612c09dd4819000 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/962853e7b2b94402b612c09dd4819000 2024-12-08T21:16:47,844 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into 962853e7b2b94402b612c09dd4819000(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:47,845 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:47,845 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=13, startTime=1733692607809; duration=0sec 2024-12-08T21:16:47,845 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:47,845 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:16:47,845 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:47,845 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:47,845 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:16:47,845 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:47,846 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/e3d41a1833b94d0d93f22476310188c6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c4ddc743aba24e458092394da6da338e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9c460361a84f4467b620ee6897268737] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.2 K 2024-12-08T21:16:47,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742342_1518 (size=4469) 2024-12-08T21:16:47,846 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d41a1833b94d0d93f22476310188c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733692604608 2024-12-08T21:16:47,847 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#447 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:47,847 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c4ddc743aba24e458092394da6da338e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733692604644 2024-12-08T21:16:47,847 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/a1d0ebac45d04d78a08cd3c448d07c4b is 175, key is test_row_0/A:col10/1733692606801/Put/seqid=0 2024-12-08T21:16:47,848 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c460361a84f4467b620ee6897268737, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733692606801 2024-12-08T21:16:47,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742343_1519 (size=31058) 2024-12-08T21:16:47,856 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:47,856 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/05351aafce17476b998539e4361c25cb is 50, key is test_row_0/C:col10/1733692606801/Put/seqid=0 2024-12-08T21:16:47,857 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/a1d0ebac45d04d78a08cd3c448d07c4b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/a1d0ebac45d04d78a08cd3c448d07c4b 2024-12-08T21:16:47,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742344_1520 (size=12104) 2024-12-08T21:16:47,862 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into a1d0ebac45d04d78a08cd3c448d07c4b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:47,862 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:47,862 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=13, startTime=1733692607809; duration=0sec 2024-12-08T21:16:47,862 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:47,862 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:16:48,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:48,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-08T21:16:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692606932, max timestamp=1733692608079Num uniques 859; 2024-12-08T21:16:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692606932, max timestamp=1733692608079Num uniques 859; 2024-12-08T21:16:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692606932, max timestamp=1733692608079Num uniques 859; 2024-12-08T21:16:48,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:48,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082a04a4e44ebf4c10aeaca1c6ccf33e32_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692606935/Put/seqid=0 2024-12-08T21:16:48,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742345_1521 (size=12154) 2024-12-08T21:16:48,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:48,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082a04a4e44ebf4c10aeaca1c6ccf33e32_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082a04a4e44ebf4c10aeaca1c6ccf33e32_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:48,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8b6f1ff52ad6479a93cbb1a852be3197, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:48,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8b6f1ff52ad6479a93cbb1a852be3197 is 175, key is test_row_0/A:col10/1733692606935/Put/seqid=0 2024-12-08T21:16:48,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742346_1522 (size=30955) 2024-12-08T21:16:48,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692608096, max timestamp=1733692608105Num uniques 859; 2024-12-08T21:16:48,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692608096, max timestamp=1733692608105Num uniques 859; 2024-12-08T21:16:48,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:48,107 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692668102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692608096, max timestamp=1733692608105Num uniques 859; 2024-12-08T21:16:48,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:16:48,108 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8b6f1ff52ad6479a93cbb1a852be3197 2024-12-08T21:16:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/29027d1c64b34364bc75b7560d30b698 is 50, key is test_row_0/B:col10/1733692606935/Put/seqid=0 2024-12-08T21:16:48,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742347_1523 (size=12001) 2024-12-08T21:16:48,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/29027d1c64b34364bc75b7560d30b698 2024-12-08T21:16:48,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ca1adc0026c44d198001c5b28fb02d2a is 50, key is test_row_0/C:col10/1733692606935/Put/seqid=0 2024-12-08T21:16:48,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742348_1524 (size=12001) 2024-12-08T21:16:48,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ca1adc0026c44d198001c5b28fb02d2a 2024-12-08T21:16:48,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8b6f1ff52ad6479a93cbb1a852be3197 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8b6f1ff52ad6479a93cbb1a852be3197 2024-12-08T21:16:48,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8b6f1ff52ad6479a93cbb1a852be3197, entries=150, sequenceid=88, filesize=30.2 K 2024-12-08T21:16:48,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692608121, max timestamp=1733692608136Num uniques 1018; 2024-12-08T21:16:48,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T21:16:48,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692608121, max timestamp=1733692608136Num uniques 1018; 2024-12-08T21:16:48,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T21:16:48,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/29027d1c64b34364bc75b7560d30b698 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/29027d1c64b34364bc75b7560d30b698 2024-12-08T21:16:48,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733692608121, max timestamp=1733692608136Num uniques 1018; 2024-12-08T21:16:48,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-08T21:16:48,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/29027d1c64b34364bc75b7560d30b698, entries=150, sequenceid=88, filesize=11.7 K 2024-12-08T21:16:48,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ca1adc0026c44d198001c5b28fb02d2a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ca1adc0026c44d198001c5b28fb02d2a 2024-12-08T21:16:48,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ca1adc0026c44d198001c5b28fb02d2a, entries=150, sequenceid=88, filesize=11.7 K 2024-12-08T21:16:48,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=73.80 KB/75570 for 0bad5c3cd9edbcf653e271e4dc44fdec in 69ms, sequenceid=88, compaction requested=false 2024-12-08T21:16:48,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:48,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=87.22 KB heapSize=240 KB 2024-12-08T21:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:48,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:48,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e4aba6c8b41a43ee908f247ac49a8972_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692608145/Put/seqid=0 2024-12-08T21:16:48,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742349_1525 (size=17284) 2024-12-08T21:16:48,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:48,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e4aba6c8b41a43ee908f247ac49a8972_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e4aba6c8b41a43ee908f247ac49a8972_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:48,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/27db3734505046b98eda0e0cf08963f0, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:48,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/27db3734505046b98eda0e0cf08963f0 is 175, key is test_row_0/A:col10/1733692608145/Put/seqid=0 2024-12-08T21:16:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692668171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692668172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692668172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692668173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742350_1526 (size=48389) 2024-12-08T21:16:48,192 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/27db3734505046b98eda0e0cf08963f0 2024-12-08T21:16:48,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/044628506d044d779f4c3bbf9371a029 is 50, key is test_row_0/B:col10/1733692608145/Put/seqid=0 2024-12-08T21:16:48,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742351_1527 (size=12151) 2024-12-08T21:16:48,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692668218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,264 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/05351aafce17476b998539e4361c25cb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/05351aafce17476b998539e4361c25cb 2024-12-08T21:16:48,269 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into 05351aafce17476b998539e4361c25cb(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:48,269 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:48,269 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=13, startTime=1733692607809; duration=0sec 2024-12-08T21:16:48,269 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:48,269 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:16:48,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692668278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692668278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692668288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692668288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692668428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692668488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692668488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692668498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692668498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/044628506d044d779f4c3bbf9371a029 2024-12-08T21:16:48,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/f1878753797c47b28d5db52cb7d9782e is 50, key is test_row_0/C:col10/1733692608145/Put/seqid=0 2024-12-08T21:16:48,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742352_1528 (size=12151) 2024-12-08T21:16:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692668738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-08T21:16:48,767 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:48,768 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-08T21:16:48,769 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T21:16:48,774 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:48,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692668798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692668798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692668808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692668808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:48,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T21:16:48,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T21:16:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:48,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:49,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/f1878753797c47b28d5db52cb7d9782e 2024-12-08T21:16:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/27db3734505046b98eda0e0cf08963f0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/27db3734505046b98eda0e0cf08963f0 2024-12-08T21:16:49,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/27db3734505046b98eda0e0cf08963f0, entries=250, sequenceid=134, filesize=47.3 K 2024-12-08T21:16:49,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/044628506d044d779f4c3bbf9371a029 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/044628506d044d779f4c3bbf9371a029 2024-12-08T21:16:49,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/044628506d044d779f4c3bbf9371a029, entries=150, sequenceid=134, filesize=11.9 K 2024-12-08T21:16:49,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/f1878753797c47b28d5db52cb7d9782e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f1878753797c47b28d5db52cb7d9782e 2024-12-08T21:16:49,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f1878753797c47b28d5db52cb7d9782e, entries=150, sequenceid=134, filesize=11.9 K 2024-12-08T21:16:49,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~239.95 KB/245712, currentSize=93.93 KB/96180 for 0bad5c3cd9edbcf653e271e4dc44fdec in 877ms, sequenceid=134, compaction requested=true 2024-12-08T21:16:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:49,029 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:49,029 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:49,030 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110402 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:49,030 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:49,030 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:16:49,030 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:16:49,030 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:49,030 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:49,030 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/962853e7b2b94402b612c09dd4819000, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/29027d1c64b34364bc75b7560d30b698, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/044628506d044d779f4c3bbf9371a029] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.4 K 2024-12-08T21:16:49,030 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/a1d0ebac45d04d78a08cd3c448d07c4b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8b6f1ff52ad6479a93cbb1a852be3197, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/27db3734505046b98eda0e0cf08963f0] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=107.8 K 2024-12-08T21:16:49,030 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:49,030 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/a1d0ebac45d04d78a08cd3c448d07c4b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8b6f1ff52ad6479a93cbb1a852be3197, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/27db3734505046b98eda0e0cf08963f0] 2024-12-08T21:16:49,030 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 962853e7b2b94402b612c09dd4819000, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733692606801 2024-12-08T21:16:49,031 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d0ebac45d04d78a08cd3c448d07c4b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733692606801 2024-12-08T21:16:49,031 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 29027d1c64b34364bc75b7560d30b698, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733692606935 2024-12-08T21:16:49,031 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b6f1ff52ad6479a93cbb1a852be3197, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733692606935 2024-12-08T21:16:49,031 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 044628506d044d779f4c3bbf9371a029, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733692608145 2024-12-08T21:16:49,031 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27db3734505046b98eda0e0cf08963f0, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733692608145 2024-12-08T21:16:49,036 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:49,037 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:49,038 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/55df279e46be4445a47d27f1ff45dba7 is 50, key is test_row_0/B:col10/1733692608145/Put/seqid=0 2024-12-08T21:16:49,038 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208746ae5651c864b9abb50beef7eb97a31_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:49,039 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208746ae5651c864b9abb50beef7eb97a31_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:49,039 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208746ae5651c864b9abb50beef7eb97a31_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:49,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742353_1529 (size=12357) 2024-12-08T21:16:49,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742354_1530 (size=4469) 2024-12-08T21:16:49,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-08T21:16:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:49,079 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T21:16:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:49,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:49,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208eaa0595804544ce4a0153df982cc247b_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692608171/Put/seqid=0 2024-12-08T21:16:49,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742355_1531 (size=12304) 2024-12-08T21:16:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T21:16:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:49,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692669297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692669308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692669308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692669318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692669318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T21:16:49,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692669408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,446 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#456 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:49,447 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8e1aac31438c4e11be7a921b306272af is 175, key is test_row_0/A:col10/1733692608145/Put/seqid=0 2024-12-08T21:16:49,449 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/55df279e46be4445a47d27f1ff45dba7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/55df279e46be4445a47d27f1ff45dba7 2024-12-08T21:16:49,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742356_1532 (size=31311) 2024-12-08T21:16:49,457 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into 55df279e46be4445a47d27f1ff45dba7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:49,457 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:49,457 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=13, startTime=1733692609029; duration=0sec 2024-12-08T21:16:49,457 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:49,457 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:16:49,458 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:49,460 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8e1aac31438c4e11be7a921b306272af as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8e1aac31438c4e11be7a921b306272af 2024-12-08T21:16:49,460 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:49,460 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:16:49,460 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:49,460 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/05351aafce17476b998539e4361c25cb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ca1adc0026c44d198001c5b28fb02d2a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f1878753797c47b28d5db52cb7d9782e] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.4 K 2024-12-08T21:16:49,460 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 05351aafce17476b998539e4361c25cb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733692606801 2024-12-08T21:16:49,460 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1adc0026c44d198001c5b28fb02d2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1733692606935 2024-12-08T21:16:49,461 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f1878753797c47b28d5db52cb7d9782e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733692608145 2024-12-08T21:16:49,463 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into 8e1aac31438c4e11be7a921b306272af(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:49,463 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:49,463 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=13, startTime=1733692609029; duration=0sec 2024-12-08T21:16:49,463 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:49,463 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:16:49,465 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#458 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:49,466 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/bd591bfaf8d44ba68d67215909bdba98 is 50, key is test_row_0/C:col10/1733692608145/Put/seqid=0 2024-12-08T21:16:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742357_1533 (size=12357) 2024-12-08T21:16:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:49,490 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208eaa0595804544ce4a0153df982cc247b_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208eaa0595804544ce4a0153df982cc247b_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:49,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/cbf746051c3f4a6881daf6a9ff80f69b, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:49,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/cbf746051c3f4a6881daf6a9ff80f69b is 175, key is test_row_0/A:col10/1733692608171/Put/seqid=0 2024-12-08T21:16:49,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742358_1534 (size=31105) 2024-12-08T21:16:49,495 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/cbf746051c3f4a6881daf6a9ff80f69b 2024-12-08T21:16:49,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4825ca7b908546749f4a592538ebbbd2 is 50, key is test_row_0/B:col10/1733692608171/Put/seqid=0 2024-12-08T21:16:49,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742359_1535 (size=12151) 2024-12-08T21:16:49,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692669617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,872 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/bd591bfaf8d44ba68d67215909bdba98 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bd591bfaf8d44ba68d67215909bdba98 2024-12-08T21:16:49,875 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into bd591bfaf8d44ba68d67215909bdba98(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:49,875 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:49,875 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=13, startTime=1733692609029; duration=0sec 2024-12-08T21:16:49,875 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:49,875 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:16:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T21:16:49,914 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4825ca7b908546749f4a592538ebbbd2 2024-12-08T21:16:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/9baa1688432e4161bd0ba9312d8909c9 is 50, key is test_row_0/C:col10/1733692608171/Put/seqid=0 2024-12-08T21:16:49,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742360_1536 (size=12151) 2024-12-08T21:16:49,923 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/9baa1688432e4161bd0ba9312d8909c9 2024-12-08T21:16:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/cbf746051c3f4a6881daf6a9ff80f69b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/cbf746051c3f4a6881daf6a9ff80f69b 2024-12-08T21:16:49,930 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/cbf746051c3f4a6881daf6a9ff80f69b, entries=150, sequenceid=152, filesize=30.4 K 2024-12-08T21:16:49,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692669928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:49,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4825ca7b908546749f4a592538ebbbd2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4825ca7b908546749f4a592538ebbbd2 2024-12-08T21:16:49,934 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4825ca7b908546749f4a592538ebbbd2, entries=150, sequenceid=152, filesize=11.9 K 2024-12-08T21:16:49,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/9baa1688432e4161bd0ba9312d8909c9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9baa1688432e4161bd0ba9312d8909c9 2024-12-08T21:16:49,938 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9baa1688432e4161bd0ba9312d8909c9, entries=150, sequenceid=152, filesize=11.9 K 2024-12-08T21:16:49,939 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0bad5c3cd9edbcf653e271e4dc44fdec in 859ms, sequenceid=152, compaction requested=false 2024-12-08T21:16:49,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:49,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:49,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-08T21:16:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-08T21:16:49,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-08T21:16:49,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1660 sec 2024-12-08T21:16:49,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.1740 sec 2024-12-08T21:16:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:50,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-08T21:16:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:50,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:50,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208140de1746a974ab2a0727105b2dc0bc3_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692609286/Put/seqid=0 2024-12-08T21:16:50,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742361_1537 (size=14794) 2024-12-08T21:16:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692670339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692670342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692670342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692670343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692670438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692670447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692670447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692670448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692670448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,450 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692670657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692670657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692670657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692670658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,731 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:50,734 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208140de1746a974ab2a0727105b2dc0bc3_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208140de1746a974ab2a0727105b2dc0bc3_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:50,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/b896501117e541e8bbd315a46089aed3, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:50,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/b896501117e541e8bbd315a46089aed3 is 175, key is test_row_0/A:col10/1733692609286/Put/seqid=0 2024-12-08T21:16:50,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742362_1538 (size=39749) 2024-12-08T21:16:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-08T21:16:50,918 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:50,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-08T21:16:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T21:16:50,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:50,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:50,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:50,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692670968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692670968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692670968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692670968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T21:16:51,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T21:16:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/b896501117e541e8bbd315a46089aed3 2024-12-08T21:16:51,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/d6c5979e1960402dbc19b19116c734f9 is 50, key is test_row_0/B:col10/1733692609286/Put/seqid=0 2024-12-08T21:16:51,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742363_1539 (size=12151) 2024-12-08T21:16:51,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T21:16:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T21:16:51,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T21:16:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692671458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:51,460 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:51,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:51,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692671478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:51,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692671478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692671478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692671478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:51,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:51,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T21:16:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/d6c5979e1960402dbc19b19116c734f9 2024-12-08T21:16:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T21:16:51,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ea80be48704c42558b9920a5fbead07a is 50, key is test_row_0/C:col10/1733692609286/Put/seqid=0 2024-12-08T21:16:51,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742364_1540 (size=12151) 2024-12-08T21:16:51,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T21:16:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T21:16:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:51,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ea80be48704c42558b9920a5fbead07a 2024-12-08T21:16:51,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/b896501117e541e8bbd315a46089aed3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/b896501117e541e8bbd315a46089aed3 2024-12-08T21:16:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/b896501117e541e8bbd315a46089aed3, entries=200, sequenceid=171, filesize=38.8 K 2024-12-08T21:16:51,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/d6c5979e1960402dbc19b19116c734f9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/d6c5979e1960402dbc19b19116c734f9 2024-12-08T21:16:51,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/d6c5979e1960402dbc19b19116c734f9, entries=150, sequenceid=171, filesize=11.9 K 2024-12-08T21:16:51,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ea80be48704c42558b9920a5fbead07a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ea80be48704c42558b9920a5fbead07a 2024-12-08T21:16:51,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ea80be48704c42558b9920a5fbead07a, entries=150, sequenceid=171, filesize=11.9 K 2024-12-08T21:16:51,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1650ms, sequenceid=171, compaction requested=true 2024-12-08T21:16:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:51,970 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:51,970 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:51,973 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:51,973 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:51,973 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:16:51,973 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:16:51,973 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,973 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,973 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8e1aac31438c4e11be7a921b306272af, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/cbf746051c3f4a6881daf6a9ff80f69b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/b896501117e541e8bbd315a46089aed3] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=99.8 K 2024-12-08T21:16:51,973 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/55df279e46be4445a47d27f1ff45dba7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4825ca7b908546749f4a592538ebbbd2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/d6c5979e1960402dbc19b19116c734f9] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.8 K 2024-12-08T21:16:51,973 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,973 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8e1aac31438c4e11be7a921b306272af, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/cbf746051c3f4a6881daf6a9ff80f69b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/b896501117e541e8bbd315a46089aed3] 2024-12-08T21:16:51,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 55df279e46be4445a47d27f1ff45dba7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733692608145 2024-12-08T21:16:51,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e1aac31438c4e11be7a921b306272af, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733692608145 2024-12-08T21:16:51,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4825ca7b908546749f4a592538ebbbd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733692608170 2024-12-08T21:16:51,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf746051c3f4a6881daf6a9ff80f69b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733692608170 2024-12-08T21:16:51,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c5979e1960402dbc19b19116c734f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733692609256 2024-12-08T21:16:51,974 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting b896501117e541e8bbd315a46089aed3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733692609256 2024-12-08T21:16:51,978 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:51,980 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120883d058e45bf54f10a067ae33d7b4c90b_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:51,980 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#464 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:51,980 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/83f37fe5504640879f57268cf67aa360 is 50, key is test_row_0/B:col10/1733692609286/Put/seqid=0 2024-12-08T21:16:51,981 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120883d058e45bf54f10a067ae33d7b4c90b_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:51,981 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120883d058e45bf54f10a067ae33d7b4c90b_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:51,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-08T21:16:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:51,996 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-08T21:16:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:51,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:51,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742365_1541 (size=12459) 2024-12-08T21:16:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084e0cdba1d56a40e9914e24f5a52d5e73_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692610342/Put/seqid=0 2024-12-08T21:16:52,004 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/83f37fe5504640879f57268cf67aa360 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/83f37fe5504640879f57268cf67aa360 2024-12-08T21:16:52,008 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into 83f37fe5504640879f57268cf67aa360(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:52,008 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:52,009 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=13, startTime=1733692611970; duration=0sec 2024-12-08T21:16:52,009 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:52,009 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:16:52,009 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:52,010 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:52,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742367_1543 (size=12304) 2024-12-08T21:16:52,011 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:16:52,011 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:52,011 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bd591bfaf8d44ba68d67215909bdba98, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9baa1688432e4161bd0ba9312d8909c9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ea80be48704c42558b9920a5fbead07a] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.8 K 2024-12-08T21:16:52,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,011 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting bd591bfaf8d44ba68d67215909bdba98, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733692608145 2024-12-08T21:16:52,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742366_1542 (size=4469) 2024-12-08T21:16:52,012 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 9baa1688432e4161bd0ba9312d8909c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1733692608170 2024-12-08T21:16:52,013 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ea80be48704c42558b9920a5fbead07a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733692609256 2024-12-08T21:16:52,013 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#465 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:52,014 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/02663c3f93454ef2b522364dfe796614 is 175, key is test_row_0/A:col10/1733692609286/Put/seqid=0 2024-12-08T21:16:52,017 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412084e0cdba1d56a40e9914e24f5a52d5e73_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084e0cdba1d56a40e9914e24f5a52d5e73_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:52,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/c1030b2093d14679a92260cf99f4bf12, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:52,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/c1030b2093d14679a92260cf99f4bf12 is 175, key is test_row_0/A:col10/1733692610342/Put/seqid=0 2024-12-08T21:16:52,021 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:52,021 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/39ad0c55ce3f4939b64256f805d409f7 is 50, key is test_row_0/C:col10/1733692609286/Put/seqid=0 2024-12-08T21:16:52,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742368_1544 (size=31413) 2024-12-08T21:16:52,026 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/02663c3f93454ef2b522364dfe796614 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/02663c3f93454ef2b522364dfe796614 2024-12-08T21:16:52,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742369_1545 (size=31105) 2024-12-08T21:16:52,027 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/c1030b2093d14679a92260cf99f4bf12 2024-12-08T21:16:52,032 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into 02663c3f93454ef2b522364dfe796614(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:52,032 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:52,032 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=13, startTime=1733692611970; duration=0sec 2024-12-08T21:16:52,032 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:52,032 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:16:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/19d5591381da4691b982443f3ff645ff is 50, key is test_row_0/B:col10/1733692610342/Put/seqid=0 2024-12-08T21:16:52,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742370_1546 (size=12459) 2024-12-08T21:16:52,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742371_1547 (size=12151) 2024-12-08T21:16:52,039 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/39ad0c55ce3f4939b64256f805d409f7 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/39ad0c55ce3f4939b64256f805d409f7 2024-12-08T21:16:52,040 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/19d5591381da4691b982443f3ff645ff 2024-12-08T21:16:52,044 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into 39ad0c55ce3f4939b64256f805d409f7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:52,044 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:52,044 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=13, startTime=1733692611970; duration=0sec 2024-12-08T21:16:52,044 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:52,044 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:16:52,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/75e3a99c0bac494dbe72f1f5bf6e9a9a is 50, key is test_row_0/C:col10/1733692610342/Put/seqid=0 2024-12-08T21:16:52,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742372_1548 (size=12151) 2024-12-08T21:16:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T21:16:52,451 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/75e3a99c0bac494dbe72f1f5bf6e9a9a 2024-12-08T21:16:52,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/c1030b2093d14679a92260cf99f4bf12 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/c1030b2093d14679a92260cf99f4bf12 2024-12-08T21:16:52,463 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/c1030b2093d14679a92260cf99f4bf12, entries=150, sequenceid=188, filesize=30.4 K 2024-12-08T21:16:52,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/19d5591381da4691b982443f3ff645ff as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/19d5591381da4691b982443f3ff645ff 2024-12-08T21:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,466 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/19d5591381da4691b982443f3ff645ff, entries=150, sequenceid=188, filesize=11.9 K 2024-12-08T21:16:52,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/75e3a99c0bac494dbe72f1f5bf6e9a9a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/75e3a99c0bac494dbe72f1f5bf6e9a9a 2024-12-08T21:16:52,469 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/75e3a99c0bac494dbe72f1f5bf6e9a9a, entries=150, sequenceid=188, filesize=11.9 K 2024-12-08T21:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,470 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 0bad5c3cd9edbcf653e271e4dc44fdec in 474ms, sequenceid=188, compaction requested=false 2024-12-08T21:16:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:52,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-08T21:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-08T21:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-08T21:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-12-08T21:16:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.5580 sec 2024-12-08T21:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:52,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:52,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c1df3ea64a0a48278ed6ae76404803fa_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692612503/Put/seqid=0 2024-12-08T21:16:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742373_1549 (size=14794) 2024-12-08T21:16:52,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692612550, max timestamp=1733692612566Num uniques 859; 2024-12-08T21:16:52,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692612550, max timestamp=1733692612566Num uniques 859; 2024-12-08T21:16:52,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:16:52,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:16:52,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692612550, max timestamp=1733692612566Num uniques 859; 2024-12-08T21:16:52,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:16:52,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692672583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692672585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692672586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692672587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692672698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692672698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692672698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692672698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692672908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692672908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692672908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692672908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:52,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:52,945 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:52,950 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c1df3ea64a0a48278ed6ae76404803fa_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c1df3ea64a0a48278ed6ae76404803fa_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:52,951 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/70d892f870b44260a71171a13664a764, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:52,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/70d892f870b44260a71171a13664a764 is 175, key is test_row_0/A:col10/1733692612503/Put/seqid=0 2024-12-08T21:16:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742374_1550 (size=39749) 2024-12-08T21:16:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-08T21:16:53,068 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:53,068 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-08T21:16:53,070 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:53,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T21:16:53,071 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:53,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:53,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T21:16:53,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692673218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692673218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692673218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692673219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:53,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:53,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,362 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/70d892f870b44260a71171a13664a764 2024-12-08T21:16:53,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:53,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T21:16:53,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/adaad665683e49c9a0f726887ad6afcc is 50, key is test_row_0/B:col10/1733692612503/Put/seqid=0 2024-12-08T21:16:53,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742375_1551 (size=12151) 2024-12-08T21:16:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692673468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,470 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:53,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T21:16:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692673728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692673728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692673728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692673728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:53,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:53,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/adaad665683e49c9a0f726887ad6afcc 2024-12-08T21:16:53,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/69c72173e0d146d9afe435960d88d624 is 50, key is test_row_0/C:col10/1733692612503/Put/seqid=0 2024-12-08T21:16:53,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742376_1552 (size=12151) 2024-12-08T21:16:53,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:53,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:54,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:54,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:54,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:54,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:54,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:54,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:54,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:16:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T21:16:54,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/69c72173e0d146d9afe435960d88d624 2024-12-08T21:16:54,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/70d892f870b44260a71171a13664a764 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/70d892f870b44260a71171a13664a764 2024-12-08T21:16:54,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/70d892f870b44260a71171a13664a764, entries=200, sequenceid=202, filesize=38.8 K 2024-12-08T21:16:54,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/adaad665683e49c9a0f726887ad6afcc as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/adaad665683e49c9a0f726887ad6afcc 2024-12-08T21:16:54,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/adaad665683e49c9a0f726887ad6afcc, entries=150, sequenceid=202, filesize=11.9 K 2024-12-08T21:16:54,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/69c72173e0d146d9afe435960d88d624 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/69c72173e0d146d9afe435960d88d624 2024-12-08T21:16:54,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/69c72173e0d146d9afe435960d88d624, entries=150, sequenceid=202, filesize=11.9 K 2024-12-08T21:16:54,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1726ms, sequenceid=202, compaction requested=true 2024-12-08T21:16:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:54,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:54,257 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:54,257 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:54,258 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:54,258 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:54,258 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:16:54,258 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:16:54,258 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:54,258 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:54,258 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/83f37fe5504640879f57268cf67aa360, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/19d5591381da4691b982443f3ff645ff, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/adaad665683e49c9a0f726887ad6afcc] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.9 K 2024-12-08T21:16:54,258 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/02663c3f93454ef2b522364dfe796614, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/c1030b2093d14679a92260cf99f4bf12, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/70d892f870b44260a71171a13664a764] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=99.9 K 2024-12-08T21:16:54,258 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:54,258 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/02663c3f93454ef2b522364dfe796614, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/c1030b2093d14679a92260cf99f4bf12, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/70d892f870b44260a71171a13664a764] 2024-12-08T21:16:54,259 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f37fe5504640879f57268cf67aa360, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733692609256 2024-12-08T21:16:54,259 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d5591381da4691b982443f3ff645ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733692610333 2024-12-08T21:16:54,259 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02663c3f93454ef2b522364dfe796614, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733692609256 2024-12-08T21:16:54,259 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting adaad665683e49c9a0f726887ad6afcc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733692612503 2024-12-08T21:16:54,259 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1030b2093d14679a92260cf99f4bf12, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733692610333 2024-12-08T21:16:54,259 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70d892f870b44260a71171a13664a764, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733692612503 2024-12-08T21:16:54,263 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:54,264 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208467cf7ac31c8467ca0ae2d0a2374c67d_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:54,265 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:54,266 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/43ebdda5e5e745c3b76da851abddc53c is 50, key is test_row_0/B:col10/1733692612503/Put/seqid=0 2024-12-08T21:16:54,266 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208467cf7ac31c8467ca0ae2d0a2374c67d_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:54,266 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208467cf7ac31c8467ca0ae2d0a2374c67d_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:54,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742378_1554 (size=12561) 2024-12-08T21:16:54,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742377_1553 (size=4469) 2024-12-08T21:16:54,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-08T21:16:54,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:54,296 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-08T21:16:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:54,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:16:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e3a12b1df51043d68e721cb4f87ff839_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692612586/Put/seqid=0 2024-12-08T21:16:54,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742379_1555 (size=12304) 2024-12-08T21:16:54,676 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#473 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:54,677 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/2672d1d9a6bc4fcea173b475db363b0b is 175, key is test_row_0/A:col10/1733692612503/Put/seqid=0 2024-12-08T21:16:54,679 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/43ebdda5e5e745c3b76da851abddc53c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/43ebdda5e5e745c3b76da851abddc53c 2024-12-08T21:16:54,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742380_1556 (size=31515) 2024-12-08T21:16:54,683 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into 43ebdda5e5e745c3b76da851abddc53c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:54,683 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/2672d1d9a6bc4fcea173b475db363b0b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2672d1d9a6bc4fcea173b475db363b0b 2024-12-08T21:16:54,684 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:54,684 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=13, startTime=1733692614257; duration=0sec 2024-12-08T21:16:54,684 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:54,684 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:16:54,684 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:54,684 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:54,684 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:16:54,684 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:54,685 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/39ad0c55ce3f4939b64256f805d409f7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/75e3a99c0bac494dbe72f1f5bf6e9a9a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/69c72173e0d146d9afe435960d88d624] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=35.9 K 2024-12-08T21:16:54,685 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ad0c55ce3f4939b64256f805d409f7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733692609256 2024-12-08T21:16:54,685 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e3a99c0bac494dbe72f1f5bf6e9a9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733692610333 2024-12-08T21:16:54,685 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c72173e0d146d9afe435960d88d624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733692612503 2024-12-08T21:16:54,687 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into 2672d1d9a6bc4fcea173b475db363b0b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:54,687 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:54,687 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=13, startTime=1733692614257; duration=0sec 2024-12-08T21:16:54,687 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:54,687 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:16:54,690 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:54,690 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/71657f26063740bc984dc772262925d2 is 50, key is test_row_0/C:col10/1733692612503/Put/seqid=0 2024-12-08T21:16:54,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742381_1557 (size=12561) 2024-12-08T21:16:54,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:54,732 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e3a12b1df51043d68e721cb4f87ff839_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3a12b1df51043d68e721cb4f87ff839_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:54,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/811a9147be474f4fbd794231a677ff16, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:54,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/811a9147be474f4fbd794231a677ff16 is 175, key is test_row_0/A:col10/1733692612586/Put/seqid=0 2024-12-08T21:16:54,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742382_1558 (size=31105) 2024-12-08T21:16:54,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:54,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692674749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692674752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:54,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692674753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:54,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692674753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692674858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692674858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:54,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692674867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:54,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692674867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692675068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692675068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692675078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692675078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,097 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/71657f26063740bc984dc772262925d2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/71657f26063740bc984dc772262925d2 2024-12-08T21:16:55,100 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into 71657f26063740bc984dc772262925d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:55,100 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:55,100 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=13, startTime=1733692614257; duration=0sec 2024-12-08T21:16:55,100 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:55,100 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:16:55,136 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/811a9147be474f4fbd794231a677ff16 2024-12-08T21:16:55,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/1dfa606aff9344b88601663118a77498 is 50, key is test_row_0/B:col10/1733692612586/Put/seqid=0 2024-12-08T21:16:55,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742383_1559 (size=12151) 2024-12-08T21:16:55,146 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/1dfa606aff9344b88601663118a77498 2024-12-08T21:16:55,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/378d9d1514e1435f866cceeb0b01f1f9 is 50, key is test_row_0/C:col10/1733692612586/Put/seqid=0 2024-12-08T21:16:55,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742384_1560 (size=12151) 2024-12-08T21:16:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T21:16:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692675378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692675378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692675388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692675388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,556 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/378d9d1514e1435f866cceeb0b01f1f9 2024-12-08T21:16:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/811a9147be474f4fbd794231a677ff16 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/811a9147be474f4fbd794231a677ff16 2024-12-08T21:16:55,561 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/811a9147be474f4fbd794231a677ff16, entries=150, sequenceid=236, filesize=30.4 K 2024-12-08T21:16:55,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/1dfa606aff9344b88601663118a77498 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/1dfa606aff9344b88601663118a77498 2024-12-08T21:16:55,564 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/1dfa606aff9344b88601663118a77498, entries=150, sequenceid=236, filesize=11.9 K 2024-12-08T21:16:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/378d9d1514e1435f866cceeb0b01f1f9 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/378d9d1514e1435f866cceeb0b01f1f9 2024-12-08T21:16:55,568 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/378d9d1514e1435f866cceeb0b01f1f9, entries=150, sequenceid=236, filesize=11.9 K 2024-12-08T21:16:55,568 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1273ms, sequenceid=236, compaction requested=false 2024-12-08T21:16:55,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:55,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:55,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-08T21:16:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-08T21:16:55,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-08T21:16:55,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4990 sec 2024-12-08T21:16:55,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.5040 sec 2024-12-08T21:16:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:55,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-08T21:16:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:55,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208345ab1cd9f8d4e7683d2d47d8a0a0983_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692614748/Put/seqid=0 2024-12-08T21:16:55,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742385_1561 (size=14794) 2024-12-08T21:16:55,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692675909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692675911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692675912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692675913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:55,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692676022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692676022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692676022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692676027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692676238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692676238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692676238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692676238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:56,301 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208345ab1cd9f8d4e7683d2d47d8a0a0983_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208345ab1cd9f8d4e7683d2d47d8a0a0983_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:56,302 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/206df6693e1d41dca60a441f8b2da1ca, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:56,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/206df6693e1d41dca60a441f8b2da1ca is 175, key is test_row_0/A:col10/1733692614748/Put/seqid=0 2024-12-08T21:16:56,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742386_1562 (size=39749) 2024-12-08T21:16:56,306 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/206df6693e1d41dca60a441f8b2da1ca 2024-12-08T21:16:56,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4d847c818b17470a8c7af76e097364c2 is 50, key is test_row_0/B:col10/1733692614748/Put/seqid=0 2024-12-08T21:16:56,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742387_1563 (size=12151) 2024-12-08T21:16:56,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692676548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692676548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692676548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692676548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:56,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4d847c818b17470a8c7af76e097364c2 2024-12-08T21:16:56,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/143bfa23b4614e57a164b49a07a25440 is 50, key is test_row_0/C:col10/1733692614748/Put/seqid=0 2024-12-08T21:16:56,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742388_1564 (size=12151) 2024-12-08T21:16:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692677058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:57,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692677058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:57,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692677058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:57,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692677058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:57,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/143bfa23b4614e57a164b49a07a25440 2024-12-08T21:16:57,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/206df6693e1d41dca60a441f8b2da1ca as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/206df6693e1d41dca60a441f8b2da1ca 2024-12-08T21:16:57,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/206df6693e1d41dca60a441f8b2da1ca, entries=200, sequenceid=253, filesize=38.8 K 2024-12-08T21:16:57,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4d847c818b17470a8c7af76e097364c2 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4d847c818b17470a8c7af76e097364c2 2024-12-08T21:16:57,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4d847c818b17470a8c7af76e097364c2, entries=150, sequenceid=253, filesize=11.9 K 2024-12-08T21:16:57,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/143bfa23b4614e57a164b49a07a25440 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/143bfa23b4614e57a164b49a07a25440 2024-12-08T21:16:57,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/143bfa23b4614e57a164b49a07a25440, entries=150, sequenceid=253, filesize=11.9 K 2024-12-08T21:16:57,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1251ms, sequenceid=253, compaction requested=true 2024-12-08T21:16:57,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:57,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:16:57,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:57,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:16:57,141 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:57,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:57,141 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:57,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:16:57,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:57,142 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:57,142 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:57,142 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:16:57,142 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:16:57,142 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:57,142 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:57,142 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/43ebdda5e5e745c3b76da851abddc53c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/1dfa606aff9344b88601663118a77498, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4d847c818b17470a8c7af76e097364c2] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=36.0 K 2024-12-08T21:16:57,142 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2672d1d9a6bc4fcea173b475db363b0b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/811a9147be474f4fbd794231a677ff16, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/206df6693e1d41dca60a441f8b2da1ca] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=100.0 K 2024-12-08T21:16:57,142 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:57,142 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2672d1d9a6bc4fcea173b475db363b0b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/811a9147be474f4fbd794231a677ff16, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/206df6693e1d41dca60a441f8b2da1ca] 2024-12-08T21:16:57,143 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2672d1d9a6bc4fcea173b475db363b0b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733692612503 2024-12-08T21:16:57,143 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 43ebdda5e5e745c3b76da851abddc53c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733692612503 2024-12-08T21:16:57,143 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfa606aff9344b88601663118a77498, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733692612579 2024-12-08T21:16:57,143 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 811a9147be474f4fbd794231a677ff16, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733692612579 2024-12-08T21:16:57,143 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d847c818b17470a8c7af76e097364c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692614743 2024-12-08T21:16:57,143 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 206df6693e1d41dca60a441f8b2da1ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692614743 2024-12-08T21:16:57,157 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:57,159 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:57,159 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412082d8d03a044ef427aa7e8c41e6c597aa6_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:57,159 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/cd45fea1afb64dd39fb9e4e11881783a is 50, key is test_row_0/B:col10/1733692614748/Put/seqid=0 2024-12-08T21:16:57,160 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412082d8d03a044ef427aa7e8c41e6c597aa6_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:57,160 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082d8d03a044ef427aa7e8c41e6c597aa6_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:57,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742389_1565 (size=12663) 2024-12-08T21:16:57,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742390_1566 (size=4469) 2024-12-08T21:16:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-08T21:16:57,238 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:57,238 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-08T21:16:57,240 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T21:16:57,240 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:57,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T21:16:57,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-08T21:16:57,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:57,392 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-08T21:16:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:57,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:57,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bc8d437a351a4123bd3fedfcaeb88578_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692615910/Put/seqid=0 2024-12-08T21:16:57,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742391_1567 (size=12454) 2024-12-08T21:16:57,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:57,419 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208bc8d437a351a4123bd3fedfcaeb88578_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bc8d437a351a4123bd3fedfcaeb88578_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:57,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3725c704e91548088d6bb5c135f6c9be, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:57,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3725c704e91548088d6bb5c135f6c9be is 175, key is test_row_0/A:col10/1733692615910/Put/seqid=0 2024-12-08T21:16:57,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742392_1568 (size=31255) 2024-12-08T21:16:57,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3725c704e91548088d6bb5c135f6c9be 2024-12-08T21:16:57,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/5caca958330a4b81aebcfb9c12e3f04c is 50, key is test_row_0/B:col10/1733692615910/Put/seqid=0 2024-12-08T21:16:57,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742393_1569 (size=12301) 2024-12-08T21:16:57,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692677529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T21:16:57,567 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#482 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:57,568 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/963b64e96eb44467bc11aa6e758a5a68 is 175, key is test_row_0/A:col10/1733692614748/Put/seqid=0 2024-12-08T21:16:57,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/cd45fea1afb64dd39fb9e4e11881783a as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/cd45fea1afb64dd39fb9e4e11881783a 2024-12-08T21:16:57,574 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into cd45fea1afb64dd39fb9e4e11881783a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:57,574 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:57,574 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=13, startTime=1733692617141; duration=0sec 2024-12-08T21:16:57,574 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:16:57,574 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:16:57,575 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:16:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742394_1570 (size=31617) 2024-12-08T21:16:57,577 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:16:57,577 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:16:57,577 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:57,577 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/71657f26063740bc984dc772262925d2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/378d9d1514e1435f866cceeb0b01f1f9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/143bfa23b4614e57a164b49a07a25440] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=36.0 K 2024-12-08T21:16:57,577 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 71657f26063740bc984dc772262925d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733692612503 2024-12-08T21:16:57,577 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 378d9d1514e1435f866cceeb0b01f1f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1733692612579 2024-12-08T21:16:57,577 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 143bfa23b4614e57a164b49a07a25440, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692614743 2024-12-08T21:16:57,582 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:16:57,582 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/843b18aeddac49edbaf69adeb944f182 is 50, key is test_row_0/C:col10/1733692614748/Put/seqid=0 2024-12-08T21:16:57,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742395_1571 (size=12663) 2024-12-08T21:16:57,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692677637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:57,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/5caca958330a4b81aebcfb9c12e3f04c 2024-12-08T21:16:57,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/5afcec8176c84ae491d5d928128d2d91 is 50, key is test_row_0/C:col10/1733692615910/Put/seqid=0 2024-12-08T21:16:57,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742396_1572 (size=12301) 2024-12-08T21:16:57,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692677848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:57,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T21:16:57,980 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/963b64e96eb44467bc11aa6e758a5a68 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/963b64e96eb44467bc11aa6e758a5a68 2024-12-08T21:16:57,984 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into 963b64e96eb44467bc11aa6e758a5a68(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:57,984 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:57,984 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=13, startTime=1733692617141; duration=0sec 2024-12-08T21:16:57,984 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:57,984 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:16:57,988 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/843b18aeddac49edbaf69adeb944f182 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/843b18aeddac49edbaf69adeb944f182 2024-12-08T21:16:57,991 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into 843b18aeddac49edbaf69adeb944f182(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:16:57,991 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:57,991 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=13, startTime=1733692617141; duration=0sec 2024-12-08T21:16:57,991 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:16:57,991 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:16:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692678068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692678068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692678068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:58,070 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:58,070 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:58,070 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:58,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:58,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692678078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:58,079 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:16:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:58,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692678158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:58,251 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/5afcec8176c84ae491d5d928128d2d91 2024-12-08T21:16:58,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/3725c704e91548088d6bb5c135f6c9be as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3725c704e91548088d6bb5c135f6c9be 2024-12-08T21:16:58,256 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3725c704e91548088d6bb5c135f6c9be, entries=150, sequenceid=271, filesize=30.5 K 2024-12-08T21:16:58,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/5caca958330a4b81aebcfb9c12e3f04c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/5caca958330a4b81aebcfb9c12e3f04c 2024-12-08T21:16:58,259 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/5caca958330a4b81aebcfb9c12e3f04c, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T21:16:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/5afcec8176c84ae491d5d928128d2d91 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5afcec8176c84ae491d5d928128d2d91 2024-12-08T21:16:58,263 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5afcec8176c84ae491d5d928128d2d91, entries=150, sequenceid=271, filesize=12.0 K 2024-12-08T21:16:58,263 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0bad5c3cd9edbcf653e271e4dc44fdec in 871ms, sequenceid=271, compaction requested=false 2024-12-08T21:16:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-08T21:16:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-08T21:16:58,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-08T21:16:58,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0270 sec 2024-12-08T21:16:58,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0310 sec 2024-12-08T21:16:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-08T21:16:58,378 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:16:58,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:16:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-08T21:16:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T21:16:58,380 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:16:58,381 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:16:58,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:16:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T21:16:58,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-08T21:16:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:58,534 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-08T21:16:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:16:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:16:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:16:58,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:16:58,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e3ad3c069c274310b426e69c97170fb1_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692617528/Put/seqid=0 2024-12-08T21:16:58,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742397_1573 (size=12454) 2024-12-08T21:16:58,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:16:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T21:16:58,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692678762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:58,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692678878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:58,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:16:58,990 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208e3ad3c069c274310b426e69c97170fb1_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3ad3c069c274310b426e69c97170fb1_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:16:58,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7b69384374834df0b40ebb0628064799, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:16:58,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7b69384374834df0b40ebb0628064799 is 175, key is test_row_0/A:col10/1733692617528/Put/seqid=0 2024-12-08T21:16:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T21:16:59,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742398_1574 (size=31255) 2024-12-08T21:16:59,044 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7b69384374834df0b40ebb0628064799 2024-12-08T21:16:59,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/25df19222f3b4a428bb11e1f99bdd540 is 50, key is test_row_0/B:col10/1733692617528/Put/seqid=0 2024-12-08T21:16:59,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692679088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:59,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:59,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:59,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:59,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742399_1575 (size=12301) 2024-12-08T21:16:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692679398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T21:16:59,523 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/25df19222f3b4a428bb11e1f99bdd540 2024-12-08T21:16:59,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/5c094829786a4b71b8463415974d603f is 50, key is test_row_0/C:col10/1733692617528/Put/seqid=0 2024-12-08T21:16:59,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742400_1576 (size=12301) 2024-12-08T21:16:59,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:16:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692679908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:16:59,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:16:59,932 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/5c094829786a4b71b8463415974d603f 2024-12-08T21:16:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7b69384374834df0b40ebb0628064799 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7b69384374834df0b40ebb0628064799 2024-12-08T21:16:59,940 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7b69384374834df0b40ebb0628064799, entries=150, sequenceid=288, filesize=30.5 K 2024-12-08T21:16:59,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/25df19222f3b4a428bb11e1f99bdd540 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/25df19222f3b4a428bb11e1f99bdd540 2024-12-08T21:16:59,943 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/25df19222f3b4a428bb11e1f99bdd540, entries=150, sequenceid=288, filesize=12.0 K 2024-12-08T21:16:59,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/5c094829786a4b71b8463415974d603f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5c094829786a4b71b8463415974d603f 2024-12-08T21:16:59,948 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5c094829786a4b71b8463415974d603f, entries=150, sequenceid=288, filesize=12.0 K 2024-12-08T21:16:59,948 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1414ms, sequenceid=288, compaction requested=true 2024-12-08T21:16:59,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:16:59,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:16:59,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-08T21:16:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-08T21:16:59,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-08T21:16:59,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-12-08T21:16:59,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.5740 sec 2024-12-08T21:17:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:00,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-08T21:17:00,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:00,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:00,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:00,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:00,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:00,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:00,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120837c95f909a9c4c84b3559cb2a3ea6847_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692620090/Put/seqid=0 2024-12-08T21:17:00,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742401_1577 (size=12454) 2024-12-08T21:17:00,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692680103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692680105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692680106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692680108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,110 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:17:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692680207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692680218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692680218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692680417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692680428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692680428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:00,504 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120837c95f909a9c4c84b3559cb2a3ea6847_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120837c95f909a9c4c84b3559cb2a3ea6847_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:00,505 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8d40290fa8214c4c81eadbfdb3d34d63, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:00,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8d40290fa8214c4c81eadbfdb3d34d63 is 175, key is test_row_0/A:col10/1733692620090/Put/seqid=0 2024-12-08T21:17:00,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742402_1578 (size=31255) 2024-12-08T21:17:00,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8d40290fa8214c4c81eadbfdb3d34d63 2024-12-08T21:17:00,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b62757bac46c481a93061301faad6f45 is 50, key is test_row_0/B:col10/1733692620090/Put/seqid=0 2024-12-08T21:17:00,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742403_1579 (size=12301) 2024-12-08T21:17:00,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b62757bac46c481a93061301faad6f45 2024-12-08T21:17:00,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/a46dbbcba9de45a8a64aa9b25be921ce is 50, key is test_row_0/C:col10/1733692620090/Put/seqid=0 2024-12-08T21:17:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-08T21:17:00,528 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:17:00,528 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:17:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-08T21:17:00,530 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:17:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T21:17:00,530 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:17:00,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:17:00,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742404_1580 (size=12301) 2024-12-08T21:17:00,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/a46dbbcba9de45a8a64aa9b25be921ce 2024-12-08T21:17:00,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/8d40290fa8214c4c81eadbfdb3d34d63 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8d40290fa8214c4c81eadbfdb3d34d63 2024-12-08T21:17:00,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8d40290fa8214c4c81eadbfdb3d34d63, entries=150, sequenceid=308, filesize=30.5 K 2024-12-08T21:17:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b62757bac46c481a93061301faad6f45 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b62757bac46c481a93061301faad6f45 2024-12-08T21:17:00,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b62757bac46c481a93061301faad6f45, entries=150, sequenceid=308, filesize=12.0 K 2024-12-08T21:17:00,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/a46dbbcba9de45a8a64aa9b25be921ce as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/a46dbbcba9de45a8a64aa9b25be921ce 2024-12-08T21:17:00,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/a46dbbcba9de45a8a64aa9b25be921ce, entries=150, sequenceid=308, filesize=12.0 K 2024-12-08T21:17:00,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0bad5c3cd9edbcf653e271e4dc44fdec in 455ms, sequenceid=308, compaction requested=true 2024-12-08T21:17:00,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:00,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:17:00,546 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:17:00,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:00,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:17:00,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:00,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:17:00,546 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:17:00,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:17:00,547 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:17:00,547 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:17:00,547 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:17:00,547 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:17:00,547 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:00,547 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:00,547 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/cd45fea1afb64dd39fb9e4e11881783a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/5caca958330a4b81aebcfb9c12e3f04c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/25df19222f3b4a428bb11e1f99bdd540, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b62757bac46c481a93061301faad6f45] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=48.4 K 2024-12-08T21:17:00,547 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/963b64e96eb44467bc11aa6e758a5a68, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3725c704e91548088d6bb5c135f6c9be, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7b69384374834df0b40ebb0628064799, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8d40290fa8214c4c81eadbfdb3d34d63] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=122.4 K 2024-12-08T21:17:00,547 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/963b64e96eb44467bc11aa6e758a5a68, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3725c704e91548088d6bb5c135f6c9be, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7b69384374834df0b40ebb0628064799, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8d40290fa8214c4c81eadbfdb3d34d63] 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting cd45fea1afb64dd39fb9e4e11881783a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692614743 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 963b64e96eb44467bc11aa6e758a5a68, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692614743 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5caca958330a4b81aebcfb9c12e3f04c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733692615907 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3725c704e91548088d6bb5c135f6c9be, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733692615907 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 25df19222f3b4a428bb11e1f99bdd540, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733692617513 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b62757bac46c481a93061301faad6f45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733692618760 2024-12-08T21:17:00,548 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b69384374834df0b40ebb0628064799, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733692617513 2024-12-08T21:17:00,549 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d40290fa8214c4c81eadbfdb3d34d63, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733692618760 2024-12-08T21:17:00,554 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:00,555 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208aaebc10f80bd4d5fa149020b54e3e76a_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:00,555 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:00,556 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/e6b1e3a1ea2245898aacbfbfe90b5821 is 50, key is test_row_0/B:col10/1733692620090/Put/seqid=0 2024-12-08T21:17:00,556 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208aaebc10f80bd4d5fa149020b54e3e76a_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:00,557 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208aaebc10f80bd4d5fa149020b54e3e76a_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:00,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742405_1581 (size=12949) 2024-12-08T21:17:00,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742406_1582 (size=4469) 2024-12-08T21:17:00,563 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#495 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:00,563 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/e544757eb85e43a5a1f55a31acbf5785 is 175, key is test_row_0/A:col10/1733692620090/Put/seqid=0 2024-12-08T21:17:00,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742407_1583 (size=31903) 2024-12-08T21:17:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T21:17:00,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-08T21:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:00,683 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-08T21:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:00,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208515da8c8689a46af995632d761b3dc1c_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692620093/Put/seqid=0 2024-12-08T21:17:00,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742408_1584 (size=12454) 2024-12-08T21:17:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:00,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:00,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692680754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692680755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692680756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T21:17:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692680868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692680868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692680868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692680918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:00,921 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:17:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:00,967 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/e6b1e3a1ea2245898aacbfbfe90b5821 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/e6b1e3a1ea2245898aacbfbfe90b5821 2024-12-08T21:17:00,974 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into e6b1e3a1ea2245898aacbfbfe90b5821(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:00,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:00,974 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=12, startTime=1733692620546; duration=0sec 2024-12-08T21:17:00,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:17:00,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:17:00,974 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:17:00,975 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/e544757eb85e43a5a1f55a31acbf5785 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/e544757eb85e43a5a1f55a31acbf5785 2024-12-08T21:17:00,975 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49566 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:17:00,976 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:17:00,976 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:00,976 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/843b18aeddac49edbaf69adeb944f182, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5afcec8176c84ae491d5d928128d2d91, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5c094829786a4b71b8463415974d603f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/a46dbbcba9de45a8a64aa9b25be921ce] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=48.4 K 2024-12-08T21:17:00,976 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 843b18aeddac49edbaf69adeb944f182, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1733692614743 2024-12-08T21:17:00,976 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5afcec8176c84ae491d5d928128d2d91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1733692615907 2024-12-08T21:17:00,977 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c094829786a4b71b8463415974d603f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733692617513 2024-12-08T21:17:00,977 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting a46dbbcba9de45a8a64aa9b25be921ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733692618760 2024-12-08T21:17:00,980 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into e544757eb85e43a5a1f55a31acbf5785(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:00,980 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:00,980 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=12, startTime=1733692620546; duration=0sec 2024-12-08T21:17:00,980 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:00,980 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:17:00,987 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:00,988 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/49010a59fe8d4f358abedb6f9a51c40c is 50, key is test_row_0/C:col10/1733692620090/Put/seqid=0 2024-12-08T21:17:01,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742409_1585 (size=12949) 2024-12-08T21:17:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692681078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692681078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692681078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,095 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208515da8c8689a46af995632d761b3dc1c_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208515da8c8689a46af995632d761b3dc1c_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/135c6e3d08b14830ac0e31aea2b2f83b, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:01,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/135c6e3d08b14830ac0e31aea2b2f83b is 175, key is test_row_0/A:col10/1733692620093/Put/seqid=0 2024-12-08T21:17:01,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742410_1586 (size=31255) 2024-12-08T21:17:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T21:17:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692681387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692681388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:01,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692681388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:01,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:01,407 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/49010a59fe8d4f358abedb6f9a51c40c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/49010a59fe8d4f358abedb6f9a51c40c 2024-12-08T21:17:01,410 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into 49010a59fe8d4f358abedb6f9a51c40c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:01,410 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:01,410 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=12, startTime=1733692620546; duration=0sec 2024-12-08T21:17:01,410 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:01,410 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:17:01,500 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/135c6e3d08b14830ac0e31aea2b2f83b 2024-12-08T21:17:01,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/465c32ec23494aabb235672d0882d178 is 50, key is test_row_0/B:col10/1733692620093/Put/seqid=0 2024-12-08T21:17:01,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742411_1587 (size=12301) 2024-12-08T21:17:01,515 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/465c32ec23494aabb235672d0882d178 2024-12-08T21:17:01,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ff77601abbfc48c1a184bcbe8554512d is 50, key is test_row_0/C:col10/1733692620093/Put/seqid=0 2024-12-08T21:17:01,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742412_1588 (size=12301) 2024-12-08T21:17:01,526 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ff77601abbfc48c1a184bcbe8554512d 2024-12-08T21:17:01,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/135c6e3d08b14830ac0e31aea2b2f83b as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/135c6e3d08b14830ac0e31aea2b2f83b 2024-12-08T21:17:01,537 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/135c6e3d08b14830ac0e31aea2b2f83b, entries=150, sequenceid=320, filesize=30.5 K 2024-12-08T21:17:01,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/465c32ec23494aabb235672d0882d178 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/465c32ec23494aabb235672d0882d178 2024-12-08T21:17:01,546 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/465c32ec23494aabb235672d0882d178, entries=150, sequenceid=320, filesize=12.0 K 2024-12-08T21:17:01,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/ff77601abbfc48c1a184bcbe8554512d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ff77601abbfc48c1a184bcbe8554512d 2024-12-08T21:17:01,554 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ff77601abbfc48c1a184bcbe8554512d, entries=150, sequenceid=320, filesize=12.0 K 2024-12-08T21:17:01,556 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0bad5c3cd9edbcf653e271e4dc44fdec in 872ms, sequenceid=320, compaction requested=false 2024-12-08T21:17:01,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:01,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:01,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-08T21:17:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-08T21:17:01,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-08T21:17:01,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0280 sec 2024-12-08T21:17:01,562 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0320 sec 2024-12-08T21:17:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-08T21:17:01,668 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:17:01,668 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:17:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-08T21:17:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T21:17:01,670 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:17:01,671 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:17:01,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:17:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T21:17:01,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-08T21:17:01,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:01,824 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-08T21:17:01,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:01,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:01,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208236fe081a43e498db33c81240501e725_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692620755/Put/seqid=0 2024-12-08T21:17:01,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742413_1589 (size=12454) 2024-12-08T21:17:01,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208236fe081a43e498db33c81240501e725_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208236fe081a43e498db33c81240501e725_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7ba23908802e445989efce64bddca57c, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:01,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7ba23908802e445989efce64bddca57c is 175, key is test_row_0/A:col10/1733692620755/Put/seqid=0 2024-12-08T21:17:01,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742414_1590 (size=31255) 2024-12-08T21:17:01,844 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7ba23908802e445989efce64bddca57c 2024-12-08T21:17:01,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/defb96a957874589bc78ca574bf1fd59 is 50, key is test_row_0/B:col10/1733692620755/Put/seqid=0 2024-12-08T21:17:01,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742415_1591 (size=12301) 2024-12-08T21:17:01,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/defb96a957874589bc78ca574bf1fd59 2024-12-08T21:17:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/8b2ebe81cc8c459f807a146444e349b1 is 50, key is test_row_0/C:col10/1733692620755/Put/seqid=0 2024-12-08T21:17:01,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742416_1592 (size=12301) 2024-12-08T21:17:01,870 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/8b2ebe81cc8c459f807a146444e349b1 2024-12-08T21:17:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/7ba23908802e445989efce64bddca57c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7ba23908802e445989efce64bddca57c 2024-12-08T21:17:01,879 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7ba23908802e445989efce64bddca57c, entries=150, sequenceid=343, filesize=30.5 K 2024-12-08T21:17:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/defb96a957874589bc78ca574bf1fd59 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/defb96a957874589bc78ca574bf1fd59 2024-12-08T21:17:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,883 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/defb96a957874589bc78ca574bf1fd59, entries=150, sequenceid=343, filesize=12.0 K 2024-12-08T21:17:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/8b2ebe81cc8c459f807a146444e349b1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/8b2ebe81cc8c459f807a146444e349b1 2024-12-08T21:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,887 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/8b2ebe81cc8c459f807a146444e349b1, entries=150, sequenceid=343, filesize=12.0 K 2024-12-08T21:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,888 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 0bad5c3cd9edbcf653e271e4dc44fdec in 64ms, sequenceid=343, compaction requested=true 2024-12-08T21:17:01,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:01,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:01,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-08T21:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-08T21:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-08T21:17:01,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 218 msec 2024-12-08T21:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 224 msec 2024-12-08T21:17:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:01,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-08T21:17:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:01,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-08T21:17:01,989 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:17:01,990 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-08T21:17:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-08T21:17:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T21:17:01,992 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-08T21:17:01,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b7e31416181540479f3e1b32ec0822a1_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692621928/Put/seqid=0 2024-12-08T21:17:01,993 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-08T21:17:01,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-08T21:17:01,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742417_1593 (size=12454) 2024-12-08T21:17:01,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:02,002 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208b7e31416181540479f3e1b32ec0822a1_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7e31416181540479f3e1b32ec0822a1_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:02,003 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/1b91c96030d64a1cb15ad7e2a47f17d4, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:02,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/1b91c96030d64a1cb15ad7e2a47f17d4 is 175, key is test_row_0/A:col10/1733692621928/Put/seqid=0 2024-12-08T21:17:02,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742418_1594 (size=31255) 2024-12-08T21:17:02,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692622000, max timestamp=1733692622013Num uniques 859; 2024-12-08T21:17:02,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:17:02,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692622000, max timestamp=1733692622013Num uniques 859; 2024-12-08T21:17:02,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:17:02,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733692622000, max timestamp=1733692622013Num uniques 859; 2024-12-08T21:17:02,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-08T21:17:02,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692682033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692682034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692682036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T21:17:02,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:02,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692682147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692682147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692682148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T21:17:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692682358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692682358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692682358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,415 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/1b91c96030d64a1cb15ad7e2a47f17d4 2024-12-08T21:17:02,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/629eeb66ff05400ea0835d2a652912f4 is 50, key is test_row_0/B:col10/1733692621928/Put/seqid=0 2024-12-08T21:17:02,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742419_1595 (size=12301) 2024-12-08T21:17:02,443 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-08T21:17:02,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T21:17:02,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692682668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692682668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692682668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:02,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/629eeb66ff05400ea0835d2a652912f4 2024-12-08T21:17:02,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/c76fb4683b474eb2812339f34e35b629 is 50, key is test_row_0/C:col10/1733692621928/Put/seqid=0 2024-12-08T21:17:02,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742420_1596 (size=12301) 2024-12-08T21:17:02,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:02,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:02,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:02,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:02,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44776 deadline: 1733692682938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:02,942 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:03,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:03,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:03,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:03,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T21:17:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692683177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692683177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:03,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692683177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:03,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:03,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:03,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:03,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-08T21:17:03,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/c76fb4683b474eb2812339f34e35b629 2024-12-08T21:17:03,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/1b91c96030d64a1cb15ad7e2a47f17d4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/1b91c96030d64a1cb15ad7e2a47f17d4 2024-12-08T21:17:03,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/1b91c96030d64a1cb15ad7e2a47f17d4, entries=150, sequenceid=353, filesize=30.5 K 2024-12-08T21:17:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/629eeb66ff05400ea0835d2a652912f4 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/629eeb66ff05400ea0835d2a652912f4 2024-12-08T21:17:03,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/629eeb66ff05400ea0835d2a652912f4, entries=150, sequenceid=353, filesize=12.0 K 2024-12-08T21:17:03,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/c76fb4683b474eb2812339f34e35b629 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c76fb4683b474eb2812339f34e35b629 2024-12-08T21:17:03,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c76fb4683b474eb2812339f34e35b629, entries=150, sequenceid=353, filesize=12.0 K 2024-12-08T21:17:03,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1269ms, sequenceid=353, compaction requested=true 2024-12-08T21:17:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:17:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:03,250 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:17:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:17:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:03,250 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:17:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:17:03,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:17:03,250 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:17:03,250 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:17:03,251 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,251 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,251 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/e544757eb85e43a5a1f55a31acbf5785, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/135c6e3d08b14830ac0e31aea2b2f83b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7ba23908802e445989efce64bddca57c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/1b91c96030d64a1cb15ad7e2a47f17d4] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=122.7 K 2024-12-08T21:17:03,251 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/e6b1e3a1ea2245898aacbfbfe90b5821, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/465c32ec23494aabb235672d0882d178, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/defb96a957874589bc78ca574bf1fd59, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/629eeb66ff05400ea0835d2a652912f4] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=48.7 K 2024-12-08T21:17:03,251 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/e544757eb85e43a5a1f55a31acbf5785, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/135c6e3d08b14830ac0e31aea2b2f83b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7ba23908802e445989efce64bddca57c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/1b91c96030d64a1cb15ad7e2a47f17d4] 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting e544757eb85e43a5a1f55a31acbf5785, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733692618760 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b1e3a1ea2245898aacbfbfe90b5821, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733692618760 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135c6e3d08b14830ac0e31aea2b2f83b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733692620093 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 465c32ec23494aabb235672d0882d178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733692620093 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ba23908802e445989efce64bddca57c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733692620748 2024-12-08T21:17:03,251 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting defb96a957874589bc78ca574bf1fd59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733692620748 2024-12-08T21:17:03,252 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b91c96030d64a1cb15ad7e2a47f17d4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733692621928 2024-12-08T21:17:03,252 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 629eeb66ff05400ea0835d2a652912f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733692621928 2024-12-08T21:17:03,256 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:03,258 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241208c5d83b86cd134e26ba833936876b1def_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:03,259 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:03,259 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/f95b72b597234ec7a6cdc12991e9e57c is 50, key is test_row_0/B:col10/1733692621928/Put/seqid=0 2024-12-08T21:17:03,259 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241208c5d83b86cd134e26ba833936876b1def_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:03,259 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208c5d83b86cd134e26ba833936876b1def_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:03,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742421_1597 (size=13085) 2024-12-08T21:17:03,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742422_1598 (size=4469) 2024-12-08T21:17:03,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38457 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-08T21:17:03,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,373 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-08T21:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:17:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081843ff78300644aabb4c8bde733343e0_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692622028/Put/seqid=0 2024-12-08T21:17:03,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742423_1599 (size=12454) 2024-12-08T21:17:03,674 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/f95b72b597234ec7a6cdc12991e9e57c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/f95b72b597234ec7a6cdc12991e9e57c 2024-12-08T21:17:03,678 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into f95b72b597234ec7a6cdc12991e9e57c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:03,679 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:03,679 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=12, startTime=1733692623250; duration=0sec 2024-12-08T21:17:03,679 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:17:03,679 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:17:03,679 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-08T21:17:03,680 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#506 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:03,680 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/879e00e12596433ab4f00224bc7bfffd is 175, key is test_row_0/A:col10/1733692621928/Put/seqid=0 2024-12-08T21:17:03,681 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-08T21:17:03,681 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:17:03,681 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:03,682 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/49010a59fe8d4f358abedb6f9a51c40c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ff77601abbfc48c1a184bcbe8554512d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/8b2ebe81cc8c459f807a146444e349b1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c76fb4683b474eb2812339f34e35b629] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=48.7 K 2024-12-08T21:17:03,682 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 49010a59fe8d4f358abedb6f9a51c40c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1733692618760 2024-12-08T21:17:03,683 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting ff77601abbfc48c1a184bcbe8554512d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733692620093 2024-12-08T21:17:03,683 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2ebe81cc8c459f807a146444e349b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733692620748 2024-12-08T21:17:03,683 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting c76fb4683b474eb2812339f34e35b629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733692621928 2024-12-08T21:17:03,691 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:03,692 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/b113a8088ad04959b8d3bea95c86b52c is 50, key is test_row_0/C:col10/1733692621928/Put/seqid=0 2024-12-08T21:17:03,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742424_1600 (size=32039) 2024-12-08T21:17:03,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742425_1601 (size=13085) 2024-12-08T21:17:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:03,789 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412081843ff78300644aabb4c8bde733343e0_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081843ff78300644aabb4c8bde733343e0_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:03,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/2781d3076dec402195cee505e74499ca, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:03,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/2781d3076dec402195cee505e74499ca is 175, key is test_row_0/A:col10/1733692622028/Put/seqid=0 2024-12-08T21:17:03,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742426_1602 (size=31255) 2024-12-08T21:17:03,811 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/2781d3076dec402195cee505e74499ca 2024-12-08T21:17:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b619293f90684384b0f6f8558dcafc61 is 50, key is test_row_0/B:col10/1733692622028/Put/seqid=0 2024-12-08T21:17:03,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742427_1603 (size=12301) 2024-12-08T21:17:03,822 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b619293f90684384b0f6f8558dcafc61 2024-12-08T21:17:03,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/57ab9ac93e304e2c9ec0debcc06cdcc3 is 50, key is test_row_0/C:col10/1733692622028/Put/seqid=0 2024-12-08T21:17:03,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742428_1604 (size=12301) 2024-12-08T21:17:04,096 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/879e00e12596433ab4f00224bc7bfffd as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/879e00e12596433ab4f00224bc7bfffd 2024-12-08T21:17:04,099 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into 879e00e12596433ab4f00224bc7bfffd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:04,099 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:04,099 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=12, startTime=1733692623250; duration=0sec 2024-12-08T21:17:04,099 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:04,099 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:17:04,102 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/b113a8088ad04959b8d3bea95c86b52c as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/b113a8088ad04959b8d3bea95c86b52c 2024-12-08T21:17:04,108 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into b113a8088ad04959b8d3bea95c86b52c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:04,108 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:04,108 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=12, startTime=1733692623250; duration=0sec 2024-12-08T21:17:04,108 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:04,108 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:17:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T21:17:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:04,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. as already flushing 2024-12-08T21:17:04,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692684179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:04,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44818 deadline: 1733692684188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:04,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44800 deadline: 1733692684188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:04,190 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_1' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:17:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:04,190 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_2' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:17:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:04,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44842 deadline: 1733692684198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:04,201 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to e653e208f502,38457,1733692444075 for 'test_row_0' in 0bad5c3cd9edbcf653e271e4dc44fdec of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-08T21:17:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,201 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:04,236 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/57ab9ac93e304e2c9ec0debcc06cdcc3 2024-12-08T21:17:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/2781d3076dec402195cee505e74499ca as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2781d3076dec402195cee505e74499ca 2024-12-08T21:17:04,242 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2781d3076dec402195cee505e74499ca, entries=150, sequenceid=388, filesize=30.5 K 2024-12-08T21:17:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/b619293f90684384b0f6f8558dcafc61 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b619293f90684384b0f6f8558dcafc61 2024-12-08T21:17:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,248 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b619293f90684384b0f6f8558dcafc61, entries=150, sequenceid=388, filesize=12.0 K 2024-12-08T21:17:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/57ab9ac93e304e2c9ec0debcc06cdcc3 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/57ab9ac93e304e2c9ec0debcc06cdcc3 2024-12-08T21:17:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,252 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/57ab9ac93e304e2c9ec0debcc06cdcc3, entries=150, sequenceid=388, filesize=12.0 K 2024-12-08T21:17:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,253 INFO [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 0bad5c3cd9edbcf653e271e4dc44fdec in 881ms, sequenceid=388, compaction requested=false 2024-12-08T21:17:04,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:04,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:04,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e653e208f502:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-08T21:17:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-08T21:17:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-08T21:17:04,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2610 sec 2024-12-08T21:17:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.2670 sec 2024-12-08T21:17:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-08T21:17:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:04,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120889b83678e23e48b280e724c4aca6bafc_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692624172/Put/seqid=0 2024-12-08T21:17:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742429_1605 (size=12454) 2024-12-08T21:17:04,317 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:04,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120889b83678e23e48b280e724c4aca6bafc_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120889b83678e23e48b280e724c4aca6bafc_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:04,321 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/9adf20c05eac4c0a92a9839ca1a606ca, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:04,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/9adf20c05eac4c0a92a9839ca1a606ca is 175, key is test_row_0/A:col10/1733692624172/Put/seqid=0 2024-12-08T21:17:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742430_1606 (size=31251) 2024-12-08T21:17:04,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692624351, max timestamp=1733692624373Num uniques 859; 2024-12-08T21:17:04,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:17:04,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692624351, max timestamp=1733692624373Num uniques 859; 2024-12-08T21:17:04,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:17:04,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733692624351, max timestamp=1733692624373Num uniques 859; 2024-12-08T21:17:04,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e653e208f502:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-08T21:17:04,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692684437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692684548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:04,602 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-12-08T21:17:04,602 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-12-08T21:17:04,602 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:04,602 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:04,602 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,602 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:04,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:04,602 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,602 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,603 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-12-08T21:17:04,603 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:04,603 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,603 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,603 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:04,605 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-12-08T21:17:04,605 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:04,605 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,605 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-12-08T21:17:04,605 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:04,605 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,605 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:04,605 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:04,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:04,726 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/9adf20c05eac4c0a92a9839ca1a606ca 2024-12-08T21:17:04,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4f473588fe05463485e0d7fd1037ebcb is 50, key is test_row_0/B:col10/1733692624172/Put/seqid=0 2024-12-08T21:17:04,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742431_1607 (size=9857) 2024-12-08T21:17:04,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692684758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:04,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-08T21:17:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44802 deadline: 1733692685068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 2024-12-08T21:17:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-08T21:17:05,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., hostname=e653e208f502,38457,1733692444075, seqNum=5 because the exception is null or not the one we care about 2024-12-08T21:17:05,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4f473588fe05463485e0d7fd1037ebcb 2024-12-08T21:17:05,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/2d6c59275df544099d6996d60105264f is 50, key is test_row_0/C:col10/1733692624172/Put/seqid=0 2024-12-08T21:17:05,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742432_1608 (size=9857) 2024-12-08T21:17:05,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/2d6c59275df544099d6996d60105264f 2024-12-08T21:17:05,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/9adf20c05eac4c0a92a9839ca1a606ca as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/9adf20c05eac4c0a92a9839ca1a606ca 2024-12-08T21:17:05,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/9adf20c05eac4c0a92a9839ca1a606ca, entries=150, sequenceid=402, filesize=30.5 K 2024-12-08T21:17:05,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/4f473588fe05463485e0d7fd1037ebcb as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4f473588fe05463485e0d7fd1037ebcb 2024-12-08T21:17:05,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4f473588fe05463485e0d7fd1037ebcb, entries=100, sequenceid=402, filesize=9.6 K 2024-12-08T21:17:05,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/2d6c59275df544099d6996d60105264f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/2d6c59275df544099d6996d60105264f 2024-12-08T21:17:05,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/2d6c59275df544099d6996d60105264f, entries=100, sequenceid=402, filesize=9.6 K 2024-12-08T21:17:05,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1273ms, sequenceid=402, compaction requested=true 2024-12-08T21:17:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:A, priority=-2147483648, current under compaction store size is 1 2024-12-08T21:17:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:05,569 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:17:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:B, priority=-2147483648, current under compaction store size is 2 2024-12-08T21:17:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bad5c3cd9edbcf653e271e4dc44fdec:C, priority=-2147483648, current under compaction store size is 3 2024-12-08T21:17:05,569 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:17:05,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/A is initiating minor compaction (all files) 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/B is initiating minor compaction (all files) 2024-12-08T21:17:05,570 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/A in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:05,570 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/B in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:05,570 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/879e00e12596433ab4f00224bc7bfffd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2781d3076dec402195cee505e74499ca, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/9adf20c05eac4c0a92a9839ca1a606ca] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=92.3 K 2024-12-08T21:17:05,570 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/f95b72b597234ec7a6cdc12991e9e57c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b619293f90684384b0f6f8558dcafc61, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4f473588fe05463485e0d7fd1037ebcb] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=34.4 K 2024-12-08T21:17:05,570 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. files: [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/879e00e12596433ab4f00224bc7bfffd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2781d3076dec402195cee505e74499ca, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/9adf20c05eac4c0a92a9839ca1a606ca] 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879e00e12596433ab4f00224bc7bfffd, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733692621928 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting f95b72b597234ec7a6cdc12991e9e57c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733692621928 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2781d3076dec402195cee505e74499ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733692622028 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b619293f90684384b0f6f8558dcafc61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733692622028 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9adf20c05eac4c0a92a9839ca1a606ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733692624172 2024-12-08T21:17:05,570 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f473588fe05463485e0d7fd1037ebcb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733692624172 2024-12-08T21:17:05,576 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:05,576 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/408844973eaa4b25b4b165d425b0e8ba is 50, key is test_row_0/B:col10/1733692624172/Put/seqid=0 2024-12-08T21:17:05,578 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38457 {}] regionserver.HRegion(8855): Flush requested on 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:05,580 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-12-08T21:17:05,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-08T21:17:05,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742433_1609 (size=13187) 2024-12-08T21:17:05,580 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:05,580 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:05,580 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:05,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:17:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:17:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:05,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-08T21:17:05,585 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412082e1f762b247e4437b38b72f80c468e9d_0bad5c3cd9edbcf653e271e4dc44fdec store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:05,590 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412082e1f762b247e4437b38b72f80c468e9d_0bad5c3cd9edbcf653e271e4dc44fdec, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:05,590 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412082e1f762b247e4437b38b72f80c468e9d_0bad5c3cd9edbcf653e271e4dc44fdec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:05,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208191e5f33c1c949ddab1c30485fd17596_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692625579/Put/seqid=0 2024-12-08T21:17:05,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742434_1610 (size=4469) 2024-12-08T21:17:05,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742435_1611 (size=12454) 2024-12-08T21:17:05,595 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:05,597 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208191e5f33c1c949ddab1c30485fd17596_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208191e5f33c1c949ddab1c30485fd17596_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:05,598 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/51f905a883694ba590de650b19a330db, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:05,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/51f905a883694ba590de650b19a330db is 175, key is test_row_0/A:col10/1733692625579/Put/seqid=0 2024-12-08T21:17:05,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742436_1612 (size=31255) 2024-12-08T21:17:05,984 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/408844973eaa4b25b4b165d425b0e8ba as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/408844973eaa4b25b4b165d425b0e8ba 2024-12-08T21:17:05,987 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/B of 0bad5c3cd9edbcf653e271e4dc44fdec into 408844973eaa4b25b4b165d425b0e8ba(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:05,987 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:05,987 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/B, priority=13, startTime=1733692625569; duration=0sec 2024-12-08T21:17:05,987 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-08T21:17:05,987 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:B 2024-12-08T21:17:05,988 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-08T21:17:05,988 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-08T21:17:05,988 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1541): 0bad5c3cd9edbcf653e271e4dc44fdec/C is initiating minor compaction (all files) 2024-12-08T21:17:05,988 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0bad5c3cd9edbcf653e271e4dc44fdec/C in TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:05,989 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/b113a8088ad04959b8d3bea95c86b52c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/57ab9ac93e304e2c9ec0debcc06cdcc3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/2d6c59275df544099d6996d60105264f] into tmpdir=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp, totalSize=34.4 K 2024-12-08T21:17:05,989 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting b113a8088ad04959b8d3bea95c86b52c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733692621928 2024-12-08T21:17:05,989 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 57ab9ac93e304e2c9ec0debcc06cdcc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1733692622028 2024-12-08T21:17:05,989 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d6c59275df544099d6996d60105264f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1733692624172 2024-12-08T21:17:05,995 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#C#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:05,995 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bad5c3cd9edbcf653e271e4dc44fdec#A#compaction#516 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-08T21:17:05,995 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/f2a24870436242d0924276aeffc93e75 is 50, key is test_row_0/C:col10/1733692624172/Put/seqid=0 2024-12-08T21:17:05,996 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/aef2578e24334a3d8236ae02c2d289e0 is 175, key is test_row_0/A:col10/1733692624172/Put/seqid=0 2024-12-08T21:17:05,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742437_1613 (size=13187) 2024-12-08T21:17:05,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742438_1614 (size=32248) 2024-12-08T21:17:06,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/51f905a883694ba590de650b19a330db 2024-12-08T21:17:06,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/0a577ad98288412a857e6474fdd1440e is 50, key is test_row_0/B:col10/1733692625579/Put/seqid=0 2024-12-08T21:17:06,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742439_1615 (size=12301) 2024-12-08T21:17:06,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-08T21:17:06,159 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:17:06,213 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-12-08T21:17:06,213 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-12-08T21:17:06,213 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:06,213 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:06,213 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,213 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,213 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,213 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:06,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:06,229 INFO [Thread-2269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2269. 2024-12-08T21:17:06,229 DEBUG [Thread-2269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:06,229 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,229 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:06,403 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/f2a24870436242d0924276aeffc93e75 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f2a24870436242d0924276aeffc93e75 2024-12-08T21:17:06,403 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/aef2578e24334a3d8236ae02c2d289e0 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/aef2578e24334a3d8236ae02c2d289e0 2024-12-08T21:17:06,406 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/A of 0bad5c3cd9edbcf653e271e4dc44fdec into aef2578e24334a3d8236ae02c2d289e0(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:06,406 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0bad5c3cd9edbcf653e271e4dc44fdec/C of 0bad5c3cd9edbcf653e271e4dc44fdec into f2a24870436242d0924276aeffc93e75(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-08T21:17:06,406 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:06,406 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:06,406 INFO [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/C, priority=13, startTime=1733692625569; duration=0sec 2024-12-08T21:17:06,406 INFO [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., storeName=0bad5c3cd9edbcf653e271e4dc44fdec/A, priority=13, startTime=1733692625569; duration=0sec 2024-12-08T21:17:06,406 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:06,406 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-08T21:17:06,406 DEBUG [RS:0;e653e208f502:38457-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:A 2024-12-08T21:17:06,406 DEBUG [RS:0;e653e208f502:38457-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bad5c3cd9edbcf653e271e4dc44fdec:C 2024-12-08T21:17:06,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/0a577ad98288412a857e6474fdd1440e 2024-12-08T21:17:06,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/d528914772a34b92be077faa9cb18414 is 50, key is test_row_0/C:col10/1733692625579/Put/seqid=0 2024-12-08T21:17:06,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742440_1616 (size=12301) 2024-12-08T21:17:06,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/d528914772a34b92be077faa9cb18414 2024-12-08T21:17:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/51f905a883694ba590de650b19a330db as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/51f905a883694ba590de650b19a330db 2024-12-08T21:17:06,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/51f905a883694ba590de650b19a330db, entries=150, sequenceid=439, filesize=30.5 K 2024-12-08T21:17:06,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/0a577ad98288412a857e6474fdd1440e as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/0a577ad98288412a857e6474fdd1440e 2024-12-08T21:17:06,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/0a577ad98288412a857e6474fdd1440e, entries=150, sequenceid=439, filesize=12.0 K 2024-12-08T21:17:06,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/d528914772a34b92be077faa9cb18414 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/d528914772a34b92be077faa9cb18414 2024-12-08T21:17:06,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/d528914772a34b92be077faa9cb18414, entries=150, sequenceid=439, filesize=12.0 K 2024-12-08T21:17:06,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=20.13 KB/20610 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1331ms, sequenceid=439, compaction requested=false 2024-12-08T21:17:06,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0bad5c3cd9edbcf653e271e4dc44fdec: 2024-12-08T21:17:06,959 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-12-08T21:17:06,959 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-08T21:17:06,959 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,959 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4167 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4234 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4167 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4156 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4241 2024-12-08T21:17:06,960 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-08T21:17:06,960 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:17:06,960 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:17:06,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:06,961 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:06,961 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-08T21:17:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-08T21:17:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T21:17:06,964 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692626964"}]},"ts":"1733692626964"} 2024-12-08T21:17:06,966 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-08T21:17:06,966 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-08T21:17:06,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-08T21:17:06,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, UNASSIGN}] 2024-12-08T21:17:06,969 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, UNASSIGN 2024-12-08T21:17:06,969 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=CLOSING, regionLocation=e653e208f502,38457,1733692444075 2024-12-08T21:17:06,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, UNASSIGN because future has completed 2024-12-08T21:17:06,971 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-08T21:17:06,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075}] 2024-12-08T21:17:07,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T21:17:07,123 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:07,123 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-08T21:17:07,124 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing 0bad5c3cd9edbcf653e271e4dc44fdec, disabling compactions & flushes 2024-12-08T21:17:07,124 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:07,124 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:07,124 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. after waiting 0 ms 2024-12-08T21:17:07,124 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:07,124 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing 0bad5c3cd9edbcf653e271e4dc44fdec 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-08T21:17:07,125 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=A 2024-12-08T21:17:07,125 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:07,125 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=B 2024-12-08T21:17:07,125 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:07,125 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bad5c3cd9edbcf653e271e4dc44fdec, store=C 2024-12-08T21:17:07,125 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-08T21:17:07,137 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ecbec5ff0f8a493aaa9937901d490541_0bad5c3cd9edbcf653e271e4dc44fdec is 50, key is test_row_0/A:col10/1733692626228/Put/seqid=0 2024-12-08T21:17:07,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742441_1617 (size=12454) 2024-12-08T21:17:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T21:17:07,540 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-08T21:17:07,544 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241208ecbec5ff0f8a493aaa9937901d490541_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ecbec5ff0f8a493aaa9937901d490541_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:07,545 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/fa9f56dd028e40bf847066805351ebaf, store: [table=TestAcidGuarantees family=A region=0bad5c3cd9edbcf653e271e4dc44fdec] 2024-12-08T21:17:07,545 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/fa9f56dd028e40bf847066805351ebaf is 175, key is test_row_0/A:col10/1733692626228/Put/seqid=0 2024-12-08T21:17:07,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742442_1618 (size=31255) 2024-12-08T21:17:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T21:17:07,949 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/fa9f56dd028e40bf847066805351ebaf 2024-12-08T21:17:07,960 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/9f7997344bc74fa89ca163dac84192d1 is 50, key is test_row_0/B:col10/1733692626228/Put/seqid=0 2024-12-08T21:17:07,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742443_1619 (size=12301) 2024-12-08T21:17:08,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T21:17:08,365 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/9f7997344bc74fa89ca163dac84192d1 2024-12-08T21:17:08,374 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/bbdc6161728844f485d696d146403034 is 50, key is test_row_0/C:col10/1733692626228/Put/seqid=0 2024-12-08T21:17:08,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742444_1620 (size=12301) 2024-12-08T21:17:08,778 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/bbdc6161728844f485d696d146403034 2024-12-08T21:17:08,782 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/A/fa9f56dd028e40bf847066805351ebaf as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/fa9f56dd028e40bf847066805351ebaf 2024-12-08T21:17:08,785 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/fa9f56dd028e40bf847066805351ebaf, entries=150, sequenceid=449, filesize=30.5 K 2024-12-08T21:17:08,786 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/B/9f7997344bc74fa89ca163dac84192d1 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/9f7997344bc74fa89ca163dac84192d1 2024-12-08T21:17:08,789 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/9f7997344bc74fa89ca163dac84192d1, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T21:17:08,790 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/.tmp/C/bbdc6161728844f485d696d146403034 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bbdc6161728844f485d696d146403034 2024-12-08T21:17:08,793 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bbdc6161728844f485d696d146403034, entries=150, sequenceid=449, filesize=12.0 K 2024-12-08T21:17:08,794 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1670ms, sequenceid=449, compaction requested=true 2024-12-08T21:17:08,794 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/ea064dfd9c884f38a4ac9650c8ec34c9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/227700aeb7d0425082ea7554f5120481, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3c057df96583479c9bb6ea54ff867596, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/a1d0ebac45d04d78a08cd3c448d07c4b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8b6f1ff52ad6479a93cbb1a852be3197, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/27db3734505046b98eda0e0cf08963f0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8e1aac31438c4e11be7a921b306272af, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/cbf746051c3f4a6881daf6a9ff80f69b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/b896501117e541e8bbd315a46089aed3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/02663c3f93454ef2b522364dfe796614, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/c1030b2093d14679a92260cf99f4bf12, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/70d892f870b44260a71171a13664a764, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2672d1d9a6bc4fcea173b475db363b0b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/811a9147be474f4fbd794231a677ff16, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/206df6693e1d41dca60a441f8b2da1ca, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/963b64e96eb44467bc11aa6e758a5a68, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3725c704e91548088d6bb5c135f6c9be, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7b69384374834df0b40ebb0628064799, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/e544757eb85e43a5a1f55a31acbf5785, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8d40290fa8214c4c81eadbfdb3d34d63, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/135c6e3d08b14830ac0e31aea2b2f83b, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7ba23908802e445989efce64bddca57c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/879e00e12596433ab4f00224bc7bfffd, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/1b91c96030d64a1cb15ad7e2a47f17d4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2781d3076dec402195cee505e74499ca, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/9adf20c05eac4c0a92a9839ca1a606ca] to archive 2024-12-08T21:17:08,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:17:08,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/ea064dfd9c884f38a4ac9650c8ec34c9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/ea064dfd9c884f38a4ac9650c8ec34c9 2024-12-08T21:17:08,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/227700aeb7d0425082ea7554f5120481 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/227700aeb7d0425082ea7554f5120481 2024-12-08T21:17:08,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3c057df96583479c9bb6ea54ff867596 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3c057df96583479c9bb6ea54ff867596 2024-12-08T21:17:08,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/a1d0ebac45d04d78a08cd3c448d07c4b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/a1d0ebac45d04d78a08cd3c448d07c4b 2024-12-08T21:17:08,800 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8b6f1ff52ad6479a93cbb1a852be3197 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8b6f1ff52ad6479a93cbb1a852be3197 2024-12-08T21:17:08,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/27db3734505046b98eda0e0cf08963f0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/27db3734505046b98eda0e0cf08963f0 2024-12-08T21:17:08,802 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8e1aac31438c4e11be7a921b306272af to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8e1aac31438c4e11be7a921b306272af 2024-12-08T21:17:08,803 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/cbf746051c3f4a6881daf6a9ff80f69b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/cbf746051c3f4a6881daf6a9ff80f69b 2024-12-08T21:17:08,804 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/b896501117e541e8bbd315a46089aed3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/b896501117e541e8bbd315a46089aed3 2024-12-08T21:17:08,805 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/02663c3f93454ef2b522364dfe796614 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/02663c3f93454ef2b522364dfe796614 2024-12-08T21:17:08,806 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/c1030b2093d14679a92260cf99f4bf12 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/c1030b2093d14679a92260cf99f4bf12 2024-12-08T21:17:08,807 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/70d892f870b44260a71171a13664a764 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/70d892f870b44260a71171a13664a764 2024-12-08T21:17:08,808 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2672d1d9a6bc4fcea173b475db363b0b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2672d1d9a6bc4fcea173b475db363b0b 2024-12-08T21:17:08,809 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/811a9147be474f4fbd794231a677ff16 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/811a9147be474f4fbd794231a677ff16 2024-12-08T21:17:08,810 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/206df6693e1d41dca60a441f8b2da1ca to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/206df6693e1d41dca60a441f8b2da1ca 2024-12-08T21:17:08,811 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/963b64e96eb44467bc11aa6e758a5a68 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/963b64e96eb44467bc11aa6e758a5a68 2024-12-08T21:17:08,812 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3725c704e91548088d6bb5c135f6c9be to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/3725c704e91548088d6bb5c135f6c9be 2024-12-08T21:17:08,813 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7b69384374834df0b40ebb0628064799 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7b69384374834df0b40ebb0628064799 2024-12-08T21:17:08,814 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/e544757eb85e43a5a1f55a31acbf5785 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/e544757eb85e43a5a1f55a31acbf5785 2024-12-08T21:17:08,815 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8d40290fa8214c4c81eadbfdb3d34d63 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/8d40290fa8214c4c81eadbfdb3d34d63 2024-12-08T21:17:08,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/135c6e3d08b14830ac0e31aea2b2f83b to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/135c6e3d08b14830ac0e31aea2b2f83b 2024-12-08T21:17:08,817 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7ba23908802e445989efce64bddca57c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/7ba23908802e445989efce64bddca57c 2024-12-08T21:17:08,818 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/879e00e12596433ab4f00224bc7bfffd to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/879e00e12596433ab4f00224bc7bfffd 2024-12-08T21:17:08,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/1b91c96030d64a1cb15ad7e2a47f17d4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/1b91c96030d64a1cb15ad7e2a47f17d4 2024-12-08T21:17:08,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2781d3076dec402195cee505e74499ca to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/2781d3076dec402195cee505e74499ca 2024-12-08T21:17:08,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/9adf20c05eac4c0a92a9839ca1a606ca to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/9adf20c05eac4c0a92a9839ca1a606ca 2024-12-08T21:17:08,821 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/005b5bf541b44dd29e580af63ce21392, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/445204b19b92461fb3265d8f855951d0, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/962853e7b2b94402b612c09dd4819000, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b2449ae1495e4adf8f6952350a3e7d53, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/29027d1c64b34364bc75b7560d30b698, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/55df279e46be4445a47d27f1ff45dba7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/044628506d044d779f4c3bbf9371a029, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4825ca7b908546749f4a592538ebbbd2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/83f37fe5504640879f57268cf67aa360, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/d6c5979e1960402dbc19b19116c734f9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/19d5591381da4691b982443f3ff645ff, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/43ebdda5e5e745c3b76da851abddc53c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/adaad665683e49c9a0f726887ad6afcc, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/1dfa606aff9344b88601663118a77498, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/cd45fea1afb64dd39fb9e4e11881783a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4d847c818b17470a8c7af76e097364c2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/5caca958330a4b81aebcfb9c12e3f04c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/25df19222f3b4a428bb11e1f99bdd540, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/e6b1e3a1ea2245898aacbfbfe90b5821, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b62757bac46c481a93061301faad6f45, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/465c32ec23494aabb235672d0882d178, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/defb96a957874589bc78ca574bf1fd59, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/f95b72b597234ec7a6cdc12991e9e57c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/629eeb66ff05400ea0835d2a652912f4, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b619293f90684384b0f6f8558dcafc61, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4f473588fe05463485e0d7fd1037ebcb] to archive 2024-12-08T21:17:08,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:17:08,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/005b5bf541b44dd29e580af63ce21392 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/005b5bf541b44dd29e580af63ce21392 2024-12-08T21:17:08,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/445204b19b92461fb3265d8f855951d0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/445204b19b92461fb3265d8f855951d0 2024-12-08T21:17:08,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/962853e7b2b94402b612c09dd4819000 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/962853e7b2b94402b612c09dd4819000 2024-12-08T21:17:08,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b2449ae1495e4adf8f6952350a3e7d53 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b2449ae1495e4adf8f6952350a3e7d53 2024-12-08T21:17:08,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/29027d1c64b34364bc75b7560d30b698 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/29027d1c64b34364bc75b7560d30b698 2024-12-08T21:17:08,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/55df279e46be4445a47d27f1ff45dba7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/55df279e46be4445a47d27f1ff45dba7 2024-12-08T21:17:08,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/044628506d044d779f4c3bbf9371a029 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/044628506d044d779f4c3bbf9371a029 2024-12-08T21:17:08,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4825ca7b908546749f4a592538ebbbd2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4825ca7b908546749f4a592538ebbbd2 2024-12-08T21:17:08,830 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/83f37fe5504640879f57268cf67aa360 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/83f37fe5504640879f57268cf67aa360 2024-12-08T21:17:08,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/d6c5979e1960402dbc19b19116c734f9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/d6c5979e1960402dbc19b19116c734f9 2024-12-08T21:17:08,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/19d5591381da4691b982443f3ff645ff to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/19d5591381da4691b982443f3ff645ff 2024-12-08T21:17:08,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/43ebdda5e5e745c3b76da851abddc53c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/43ebdda5e5e745c3b76da851abddc53c 2024-12-08T21:17:08,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/adaad665683e49c9a0f726887ad6afcc to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/adaad665683e49c9a0f726887ad6afcc 2024-12-08T21:17:08,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/1dfa606aff9344b88601663118a77498 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/1dfa606aff9344b88601663118a77498 2024-12-08T21:17:08,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/cd45fea1afb64dd39fb9e4e11881783a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/cd45fea1afb64dd39fb9e4e11881783a 2024-12-08T21:17:08,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4d847c818b17470a8c7af76e097364c2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4d847c818b17470a8c7af76e097364c2 2024-12-08T21:17:08,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/5caca958330a4b81aebcfb9c12e3f04c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/5caca958330a4b81aebcfb9c12e3f04c 2024-12-08T21:17:08,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/25df19222f3b4a428bb11e1f99bdd540 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/25df19222f3b4a428bb11e1f99bdd540 2024-12-08T21:17:08,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/e6b1e3a1ea2245898aacbfbfe90b5821 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/e6b1e3a1ea2245898aacbfbfe90b5821 2024-12-08T21:17:08,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b62757bac46c481a93061301faad6f45 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b62757bac46c481a93061301faad6f45 2024-12-08T21:17:08,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/465c32ec23494aabb235672d0882d178 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/465c32ec23494aabb235672d0882d178 2024-12-08T21:17:08,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/defb96a957874589bc78ca574bf1fd59 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/defb96a957874589bc78ca574bf1fd59 2024-12-08T21:17:08,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/f95b72b597234ec7a6cdc12991e9e57c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/f95b72b597234ec7a6cdc12991e9e57c 2024-12-08T21:17:08,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/629eeb66ff05400ea0835d2a652912f4 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/629eeb66ff05400ea0835d2a652912f4 2024-12-08T21:17:08,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b619293f90684384b0f6f8558dcafc61 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/b619293f90684384b0f6f8558dcafc61 2024-12-08T21:17:08,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4f473588fe05463485e0d7fd1037ebcb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/4f473588fe05463485e0d7fd1037ebcb 2024-12-08T21:17:08,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/e3d41a1833b94d0d93f22476310188c6, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c4ddc743aba24e458092394da6da338e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/05351aafce17476b998539e4361c25cb, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9c460361a84f4467b620ee6897268737, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ca1adc0026c44d198001c5b28fb02d2a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bd591bfaf8d44ba68d67215909bdba98, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f1878753797c47b28d5db52cb7d9782e, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9baa1688432e4161bd0ba9312d8909c9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/39ad0c55ce3f4939b64256f805d409f7, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ea80be48704c42558b9920a5fbead07a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/75e3a99c0bac494dbe72f1f5bf6e9a9a, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/71657f26063740bc984dc772262925d2, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/69c72173e0d146d9afe435960d88d624, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/378d9d1514e1435f866cceeb0b01f1f9, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/843b18aeddac49edbaf69adeb944f182, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/143bfa23b4614e57a164b49a07a25440, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5afcec8176c84ae491d5d928128d2d91, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5c094829786a4b71b8463415974d603f, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/49010a59fe8d4f358abedb6f9a51c40c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/a46dbbcba9de45a8a64aa9b25be921ce, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ff77601abbfc48c1a184bcbe8554512d, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/8b2ebe81cc8c459f807a146444e349b1, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/b113a8088ad04959b8d3bea95c86b52c, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c76fb4683b474eb2812339f34e35b629, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/57ab9ac93e304e2c9ec0debcc06cdcc3, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/2d6c59275df544099d6996d60105264f] to archive 2024-12-08T21:17:08,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-08T21:17:08,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/e3d41a1833b94d0d93f22476310188c6 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/e3d41a1833b94d0d93f22476310188c6 2024-12-08T21:17:08,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c4ddc743aba24e458092394da6da338e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c4ddc743aba24e458092394da6da338e 2024-12-08T21:17:08,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/05351aafce17476b998539e4361c25cb to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/05351aafce17476b998539e4361c25cb 2024-12-08T21:17:08,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9c460361a84f4467b620ee6897268737 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9c460361a84f4467b620ee6897268737 2024-12-08T21:17:08,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ca1adc0026c44d198001c5b28fb02d2a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ca1adc0026c44d198001c5b28fb02d2a 2024-12-08T21:17:08,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bd591bfaf8d44ba68d67215909bdba98 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bd591bfaf8d44ba68d67215909bdba98 2024-12-08T21:17:08,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f1878753797c47b28d5db52cb7d9782e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f1878753797c47b28d5db52cb7d9782e 2024-12-08T21:17:08,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9baa1688432e4161bd0ba9312d8909c9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/9baa1688432e4161bd0ba9312d8909c9 2024-12-08T21:17:08,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/39ad0c55ce3f4939b64256f805d409f7 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/39ad0c55ce3f4939b64256f805d409f7 2024-12-08T21:17:08,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ea80be48704c42558b9920a5fbead07a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ea80be48704c42558b9920a5fbead07a 2024-12-08T21:17:08,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/75e3a99c0bac494dbe72f1f5bf6e9a9a to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/75e3a99c0bac494dbe72f1f5bf6e9a9a 2024-12-08T21:17:08,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/71657f26063740bc984dc772262925d2 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/71657f26063740bc984dc772262925d2 2024-12-08T21:17:08,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/69c72173e0d146d9afe435960d88d624 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/69c72173e0d146d9afe435960d88d624 2024-12-08T21:17:08,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/378d9d1514e1435f866cceeb0b01f1f9 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/378d9d1514e1435f866cceeb0b01f1f9 2024-12-08T21:17:08,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/843b18aeddac49edbaf69adeb944f182 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/843b18aeddac49edbaf69adeb944f182 2024-12-08T21:17:08,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/143bfa23b4614e57a164b49a07a25440 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/143bfa23b4614e57a164b49a07a25440 2024-12-08T21:17:08,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5afcec8176c84ae491d5d928128d2d91 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5afcec8176c84ae491d5d928128d2d91 2024-12-08T21:17:08,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5c094829786a4b71b8463415974d603f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/5c094829786a4b71b8463415974d603f 2024-12-08T21:17:08,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/49010a59fe8d4f358abedb6f9a51c40c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/49010a59fe8d4f358abedb6f9a51c40c 2024-12-08T21:17:08,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/a46dbbcba9de45a8a64aa9b25be921ce to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/a46dbbcba9de45a8a64aa9b25be921ce 2024-12-08T21:17:08,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ff77601abbfc48c1a184bcbe8554512d to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/ff77601abbfc48c1a184bcbe8554512d 2024-12-08T21:17:08,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/8b2ebe81cc8c459f807a146444e349b1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/8b2ebe81cc8c459f807a146444e349b1 2024-12-08T21:17:08,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/b113a8088ad04959b8d3bea95c86b52c to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/b113a8088ad04959b8d3bea95c86b52c 2024-12-08T21:17:08,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c76fb4683b474eb2812339f34e35b629 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/c76fb4683b474eb2812339f34e35b629 2024-12-08T21:17:08,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/57ab9ac93e304e2c9ec0debcc06cdcc3 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/57ab9ac93e304e2c9ec0debcc06cdcc3 2024-12-08T21:17:08,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/2d6c59275df544099d6996d60105264f to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/2d6c59275df544099d6996d60105264f 2024-12-08T21:17:08,875 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/recovered.edits/452.seqid, newMaxSeqId=452, maxSeqId=4 2024-12-08T21:17:08,876 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. 2024-12-08T21:17:08,876 DEBUG [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for 0bad5c3cd9edbcf653e271e4dc44fdec: Waiting for close lock at 1733692627124Running coprocessor pre-close hooks at 1733692627124Disabling compacts and flushes for region at 1733692627124Disabling writes for close at 1733692627124Obtaining lock to block concurrent updates at 1733692627124Preparing flush snapshotting stores in 0bad5c3cd9edbcf653e271e4dc44fdec at 1733692627124Finished memstore snapshotting TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733692627125 (+1 ms)Flushing stores of TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec. at 1733692627127 (+2 ms)Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/A: creating writer at 1733692627127Flushing mob file 0bad5c3cd9edbcf653e271e4dc44fdec/A: appending metadata at 1733692627137 (+10 ms)Flushing mob file 0bad5c3cd9edbcf653e271e4dc44fdec/A: closing flushed file at 1733692627137Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/A: appending metadata at 1733692627545 (+408 ms)Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/A: closing flushed file at 1733692627545Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/B: creating writer at 1733692627957 (+412 ms)Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/B: appending metadata at 1733692627960 (+3 ms)Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/B: closing flushed file at 1733692627960Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/C: creating writer at 1733692628370 (+410 ms)Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/C: appending metadata at 1733692628374 (+4 ms)Flushing 0bad5c3cd9edbcf653e271e4dc44fdec/C: closing flushed file at 1733692628374Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@836fcb0: reopening flushed file at 1733692628781 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c4d9717: reopening flushed file at 1733692628785 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67d5040a: reopening flushed file at 1733692628790 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 0bad5c3cd9edbcf653e271e4dc44fdec in 1670ms, sequenceid=449, compaction requested=true at 1733692628794 (+4 ms)Writing region close event to WAL at 1733692628873 (+79 ms)Running coprocessor post-close hooks at 1733692628875 (+2 ms)Closed at 1733692628876 (+1 ms) 2024-12-08T21:17:08,877 INFO [RS_CLOSE_REGION-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed 0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:08,878 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=0bad5c3cd9edbcf653e271e4dc44fdec, regionState=CLOSED 2024-12-08T21:17:08,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 because future has completed 2024-12-08T21:17:08,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-08T21:17:08,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure 0bad5c3cd9edbcf653e271e4dc44fdec, server=e653e208f502,38457,1733692444075 in 1.9090 sec 2024-12-08T21:17:08,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-08T21:17:08,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bad5c3cd9edbcf653e271e4dc44fdec, UNASSIGN in 1.9130 sec 2024-12-08T21:17:08,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-08T21:17:08,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9160 sec 2024-12-08T21:17:08,885 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733692628885"}]},"ts":"1733692628885"} 2024-12-08T21:17:08,886 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-08T21:17:08,886 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-08T21:17:08,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9250 sec 2024-12-08T21:17:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-08T21:17:09,119 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:17:09,120 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-08T21:17:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:17:09,124 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:17:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T21:17:09,126 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:17:09,130 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,134 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C, FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/recovered.edits] 2024-12-08T21:17:09,139 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/51f905a883694ba590de650b19a330db to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/51f905a883694ba590de650b19a330db 2024-12-08T21:17:09,141 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/aef2578e24334a3d8236ae02c2d289e0 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/aef2578e24334a3d8236ae02c2d289e0 2024-12-08T21:17:09,143 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/fa9f56dd028e40bf847066805351ebaf to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/A/fa9f56dd028e40bf847066805351ebaf 2024-12-08T21:17:09,146 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/0a577ad98288412a857e6474fdd1440e to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/0a577ad98288412a857e6474fdd1440e 2024-12-08T21:17:09,148 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/408844973eaa4b25b4b165d425b0e8ba to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/408844973eaa4b25b4b165d425b0e8ba 2024-12-08T21:17:09,150 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/9f7997344bc74fa89ca163dac84192d1 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/B/9f7997344bc74fa89ca163dac84192d1 2024-12-08T21:17:09,153 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bbdc6161728844f485d696d146403034 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/bbdc6161728844f485d696d146403034 2024-12-08T21:17:09,154 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/d528914772a34b92be077faa9cb18414 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/d528914772a34b92be077faa9cb18414 2024-12-08T21:17:09,155 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f2a24870436242d0924276aeffc93e75 to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/C/f2a24870436242d0924276aeffc93e75 2024-12-08T21:17:09,157 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/recovered.edits/452.seqid to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec/recovered.edits/452.seqid 2024-12-08T21:17:09,157 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/default/TestAcidGuarantees/0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,157 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-08T21:17:09,158 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T21:17:09,158 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-08T21:17:09,161 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208140de1746a974ab2a0727105b2dc0bc3_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208140de1746a974ab2a0727105b2dc0bc3_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,162 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081843ff78300644aabb4c8bde733343e0_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412081843ff78300644aabb4c8bde733343e0_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,163 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208191e5f33c1c949ddab1c30485fd17596_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208191e5f33c1c949ddab1c30485fd17596_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,164 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208236fe081a43e498db33c81240501e725_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208236fe081a43e498db33c81240501e725_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,165 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082a04a4e44ebf4c10aeaca1c6ccf33e32_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082a04a4e44ebf4c10aeaca1c6ccf33e32_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,167 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082f3be550ab29467ca47096482ad3b946_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412082f3be550ab29467ca47096482ad3b946_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,168 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208345ab1cd9f8d4e7683d2d47d8a0a0983_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208345ab1cd9f8d4e7683d2d47d8a0a0983_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,169 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120837c95f909a9c4c84b3559cb2a3ea6847_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120837c95f909a9c4c84b3559cb2a3ea6847_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,170 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120842f32fbb319a4d0288384e60071f9e0d_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120842f32fbb319a4d0288384e60071f9e0d_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,171 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084e0cdba1d56a40e9914e24f5a52d5e73_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412084e0cdba1d56a40e9914e24f5a52d5e73_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,171 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208515da8c8689a46af995632d761b3dc1c_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208515da8c8689a46af995632d761b3dc1c_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,172 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120889b83678e23e48b280e724c4aca6bafc_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120889b83678e23e48b280e724c4aca6bafc_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,173 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7e31416181540479f3e1b32ec0822a1_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208b7e31416181540479f3e1b32ec0822a1_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,174 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bc8d437a351a4123bd3fedfcaeb88578_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208bc8d437a351a4123bd3fedfcaeb88578_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,174 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c1df3ea64a0a48278ed6ae76404803fa_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208c1df3ea64a0a48278ed6ae76404803fa_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,175 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3a12b1df51043d68e721cb4f87ff839_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3a12b1df51043d68e721cb4f87ff839_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,176 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3ad3c069c274310b426e69c97170fb1_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e3ad3c069c274310b426e69c97170fb1_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,177 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e4aba6c8b41a43ee908f247ac49a8972_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208e4aba6c8b41a43ee908f247ac49a8972_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,177 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208eaa0595804544ce4a0153df982cc247b_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208eaa0595804544ce4a0153df982cc247b_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,178 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ecbec5ff0f8a493aaa9937901d490541_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208ecbec5ff0f8a493aaa9937901d490541_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,179 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe6d5fad22ec4f65ab31eda120c5f249_0bad5c3cd9edbcf653e271e4dc44fdec to hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241208fe6d5fad22ec4f65ab31eda120c5f249_0bad5c3cd9edbcf653e271e4dc44fdec 2024-12-08T21:17:09,179 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-08T21:17:09,181 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:17:09,183 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-08T21:17:09,184 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-08T21:17:09,185 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:17:09,185 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-08T21:17:09,185 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733692629185"}]},"ts":"9223372036854775807"} 2024-12-08T21:17:09,186 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-08T21:17:09,186 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0bad5c3cd9edbcf653e271e4dc44fdec, NAME => 'TestAcidGuarantees,,1733692601465.0bad5c3cd9edbcf653e271e4dc44fdec.', STARTKEY => '', ENDKEY => ''}] 2024-12-08T21:17:09,187 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-08T21:17:09,187 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733692629187"}]},"ts":"9223372036854775807"} 2024-12-08T21:17:09,188 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-08T21:17:09,188 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-08T21:17:09,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-12-08T21:17:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-08T21:17:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-08T21:17:09,238 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-08T21:17:09,253 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=499 (was 551), ProcessCount=11 (was 11), AvailableMemoryMB=7920 (was 8210) 2024-12-08T21:17:09,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-08T21:17:09,254 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:17:09,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:17:09,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:09,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:09,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-08T21:17:09,254 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-08T21:17:09,254 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-12-08T21:17:09,255 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e653e208f502,45981,1733692443391 2024-12-08T21:17:09,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:17:09,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:17:09,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-08T21:17:09,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:17:09,296 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:17:09,297 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-08T21:17:09,298 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:17:09,298 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:17:09,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:09,298 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-08T21:17:09,299 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e653e208f502,38457,1733692444075' ***** 2024-12-08T21:17:09,299 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-08T21:17:09,300 INFO [RS:0;e653e208f502:38457 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-08T21:17:09,300 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-08T21:17:09,300 INFO [RS:0;e653e208f502:38457 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-08T21:17:09,300 INFO [RS:0;e653e208f502:38457 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-08T21:17:09,300 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(959): stopping server e653e208f502,38457,1733692444075 2024-12-08T21:17:09,301 INFO [RS:0;e653e208f502:38457 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:17:09,301 INFO [RS:0;e653e208f502:38457 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e653e208f502:38457. 2024-12-08T21:17:09,301 DEBUG [RS:0;e653e208f502:38457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-08T21:17:09,301 DEBUG [RS:0;e653e208f502:38457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:09,301 INFO [RS:0;e653e208f502:38457 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-08T21:17:09,301 INFO [RS:0;e653e208f502:38457 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-08T21:17:09,301 INFO [RS:0;e653e208f502:38457 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-08T21:17:09,301 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-08T21:17:09,302 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-08T21:17:09,302 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-08T21:17:09,303 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-08T21:17:09,303 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-08T21:17:09,303 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-08T21:17:09,303 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:09,303 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-08T21:17:09,303 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-08T21:17:09,303 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-08T21:17:09,321 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/info/f4b7a071424142ec9e457d35df14b152 is 95, key is TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164./info:/1733692477537/DeleteFamily/seqid=0 2024-12-08T21:17:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742445_1621 (size=6514) 2024-12-08T21:17:09,349 INFO [regionserver/e653e208f502:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:17:09,503 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:09,703 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:09,726 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/info/f4b7a071424142ec9e457d35df14b152 2024-12-08T21:17:09,753 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/ns/b9ad88edb1a145e7840380cca77e4391 is 93, key is TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164./ns:/1733692477499/DeleteFamily/seqid=0 2024-12-08T21:17:09,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742446_1622 (size=5906) 2024-12-08T21:17:09,904 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:10,104 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:10,157 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/ns/b9ad88edb1a145e7840380cca77e4391 2024-12-08T21:17:10,184 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/rep_barrier/73a42afe91ee44ce9a7cbd5671a8d04d is 102, key is TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164./rep_barrier:/1733692477499/DeleteFamily/seqid=0 2024-12-08T21:17:10,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742447_1623 (size=6025) 2024-12-08T21:17:10,216 INFO [regionserver/e653e208f502:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-08T21:17:10,216 INFO [regionserver/e653e208f502:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-08T21:17:10,304 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:10,504 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:10,587 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/rep_barrier/73a42afe91ee44ce9a7cbd5671a8d04d 2024-12-08T21:17:10,603 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/table/65cd649798bb4a648c12bc65baff1ae6 is 96, key is TestAcidGuarantees,,1733692448721.0ef41e10da56e9b6857487fb3e409164./table:/1733692477499/DeleteFamily/seqid=0 2024-12-08T21:17:10,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742448_1624 (size=5936) 2024-12-08T21:17:10,705 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:10,905 DEBUG [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-08T21:17:11,008 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/table/65cd649798bb4a648c12bc65baff1ae6 2024-12-08T21:17:11,018 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/info/f4b7a071424142ec9e457d35df14b152 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/info/f4b7a071424142ec9e457d35df14b152 2024-12-08T21:17:11,022 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/info/f4b7a071424142ec9e457d35df14b152, entries=12, sequenceid=90, filesize=6.4 K 2024-12-08T21:17:11,023 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/ns/b9ad88edb1a145e7840380cca77e4391 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/ns/b9ad88edb1a145e7840380cca77e4391 2024-12-08T21:17:11,028 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/ns/b9ad88edb1a145e7840380cca77e4391, entries=8, sequenceid=90, filesize=5.8 K 2024-12-08T21:17:11,028 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/rep_barrier/73a42afe91ee44ce9a7cbd5671a8d04d as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/rep_barrier/73a42afe91ee44ce9a7cbd5671a8d04d 2024-12-08T21:17:11,032 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/rep_barrier/73a42afe91ee44ce9a7cbd5671a8d04d, entries=6, sequenceid=90, filesize=5.9 K 2024-12-08T21:17:11,033 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/.tmp/table/65cd649798bb4a648c12bc65baff1ae6 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/table/65cd649798bb4a648c12bc65baff1ae6 2024-12-08T21:17:11,036 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/table/65cd649798bb4a648c12bc65baff1ae6, entries=7, sequenceid=90, filesize=5.8 K 2024-12-08T21:17:11,037 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1734ms, sequenceid=90, compaction requested=false 2024-12-08T21:17:11,042 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-08T21:17:11,043 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-08T21:17:11,044 INFO [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-08T21:17:11,044 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733692629302Running coprocessor pre-close hooks at 1733692629303 (+1 ms)Disabling compacts and flushes for region at 1733692629303Disabling writes for close at 1733692629303Obtaining lock to block concurrent updates at 1733692629303Preparing flush snapshotting stores in 1588230740 at 1733692629303Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733692629304 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733692629304Flushing 1588230740/info: creating writer at 1733692629304Flushing 1588230740/info: appending metadata at 1733692629321 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733692629321Flushing 1588230740/ns: creating writer at 1733692629736 (+415 ms)Flushing 1588230740/ns: appending metadata at 1733692629752 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733692629752Flushing 1588230740/rep_barrier: creating writer at 1733692630166 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733692630183 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733692630183Flushing 1588230740/table: creating writer at 1733692630590 (+407 ms)Flushing 1588230740/table: appending metadata at 1733692630603 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733692630603Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b98923e: reopening flushed file at 1733692631017 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a113d72: reopening flushed file at 1733692631022 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51ca8606: reopening flushed file at 1733692631028 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7da7b65a: reopening flushed file at 1733692631032 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1734ms, sequenceid=90, compaction requested=false at 1733692631037 (+5 ms)Writing region close event to WAL at 1733692631039 (+2 ms)Running coprocessor post-close hooks at 1733692631043 (+4 ms)Closed at 1733692631043 2024-12-08T21:17:11,044 DEBUG [RS_CLOSE_META-regionserver/e653e208f502:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-08T21:17:11,105 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(976): stopping server e653e208f502,38457,1733692444075; all regions closed. 2024-12-08T21:17:11,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741834_1010 (size=25344) 2024-12-08T21:17:11,117 DEBUG [RS:0;e653e208f502:38457 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/oldWALs 2024-12-08T21:17:11,117 INFO [RS:0;e653e208f502:38457 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e653e208f502%2C38457%2C1733692444075.meta:.meta(num 1733692448297) 2024-12-08T21:17:11,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741832_1008 (size=19883680) 2024-12-08T21:17:11,122 DEBUG [RS:0;e653e208f502:38457 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/oldWALs 2024-12-08T21:17:11,122 INFO [RS:0;e653e208f502:38457 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e653e208f502%2C38457%2C1733692444075:(num 1733692447315) 2024-12-08T21:17:11,122 DEBUG [RS:0;e653e208f502:38457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-08T21:17:11,122 INFO [RS:0;e653e208f502:38457 {}] regionserver.LeaseManager(133): Closed leases 2024-12-08T21:17:11,122 INFO [RS:0;e653e208f502:38457 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:17:11,123 INFO [RS:0;e653e208f502:38457 {}] hbase.ChoreService(370): Chore service for: regionserver/e653e208f502:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-08T21:17:11,123 INFO [RS:0;e653e208f502:38457 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:17:11,123 INFO [regionserver/e653e208f502:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:17:11,124 INFO [RS:0;e653e208f502:38457 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38457 2024-12-08T21:17:11,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e653e208f502,38457,1733692444075 2024-12-08T21:17:11,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-08T21:17:11,175 INFO [RS:0;e653e208f502:38457 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:17:11,176 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e653e208f502,38457,1733692444075] 2024-12-08T21:17:11,196 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e653e208f502,38457,1733692444075 already deleted, retry=false 2024-12-08T21:17:11,197 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e653e208f502,38457,1733692444075 expired; onlineServers=0 2024-12-08T21:17:11,197 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e653e208f502,45981,1733692443391' ***** 2024-12-08T21:17:11,197 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-08T21:17:11,197 INFO [M:0;e653e208f502:45981 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-08T21:17:11,197 INFO [M:0;e653e208f502:45981 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-08T21:17:11,198 DEBUG [M:0;e653e208f502:45981 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-08T21:17:11,198 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-08T21:17:11,198 DEBUG [M:0;e653e208f502:45981 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-08T21:17:11,198 DEBUG [master/e653e208f502:0:becomeActiveMaster-HFileCleaner.small.0-1733692446978 {}] cleaner.HFileCleaner(306): Exit Thread[master/e653e208f502:0:becomeActiveMaster-HFileCleaner.small.0-1733692446978,5,FailOnTimeoutGroup] 2024-12-08T21:17:11,198 DEBUG [master/e653e208f502:0:becomeActiveMaster-HFileCleaner.large.0-1733692446977 {}] cleaner.HFileCleaner(306): Exit Thread[master/e653e208f502:0:becomeActiveMaster-HFileCleaner.large.0-1733692446977,5,FailOnTimeoutGroup] 2024-12-08T21:17:11,198 INFO [M:0;e653e208f502:45981 {}] hbase.ChoreService(370): Chore service for: master/e653e208f502:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-08T21:17:11,198 INFO [M:0;e653e208f502:45981 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-08T21:17:11,199 DEBUG [M:0;e653e208f502:45981 {}] master.HMaster(1795): Stopping service threads 2024-12-08T21:17:11,199 INFO [M:0;e653e208f502:45981 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-08T21:17:11,199 INFO [M:0;e653e208f502:45981 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-08T21:17:11,199 ERROR [M:0;e653e208f502:45981 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-08T21:17:11,200 INFO [M:0;e653e208f502:45981 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-08T21:17:11,201 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-08T21:17:11,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-08T21:17:11,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-08T21:17:11,208 DEBUG [M:0;e653e208f502:45981 {}] zookeeper.ZKUtil(347): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-08T21:17:11,208 WARN [M:0;e653e208f502:45981 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-08T21:17:11,209 INFO [M:0;e653e208f502:45981 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/.lastflushedseqids 2024-12-08T21:17:11,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742449_1625 (size=72) 2024-12-08T21:17:11,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:17:11,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38457-0x10007474e6b0001, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:17:11,288 INFO [RS:0;e653e208f502:38457 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:17:11,288 INFO [RS:0;e653e208f502:38457 {}] regionserver.HRegionServer(1031): Exiting; stopping=e653e208f502,38457,1733692444075; zookeeper connection closed. 2024-12-08T21:17:11,289 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6a448784 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6a448784 2024-12-08T21:17:11,290 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-08T21:17:11,624 INFO [M:0;e653e208f502:45981 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-08T21:17:11,624 INFO [M:0;e653e208f502:45981 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-08T21:17:11,625 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-08T21:17:11,625 INFO [M:0;e653e208f502:45981 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:17:11,625 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:17:11,625 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-08T21:17:11,625 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:17:11,625 INFO [M:0;e653e208f502:45981 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=756.89 KB heapSize=925.94 KB 2024-12-08T21:17:11,646 DEBUG [M:0;e653e208f502:45981 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b732dbe249ff42179be1b143de7c9d7f is 82, key is hbase:meta,,1/info:regioninfo/1733692448378/Put/seqid=0 2024-12-08T21:17:11,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742450_1626 (size=5672) 2024-12-08T21:17:12,055 INFO [M:0;e653e208f502:45981 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2084 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b732dbe249ff42179be1b143de7c9d7f 2024-12-08T21:17:12,081 DEBUG [M:0;e653e208f502:45981 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00af7895dc8f452fb1aee77c3b336665 is 2480, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1733692480730/Put/seqid=0 2024-12-08T21:17:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742451_1627 (size=44719) 2024-12-08T21:17:12,485 INFO [M:0;e653e208f502:45981 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=756.33 KB at sequenceid=2084 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00af7895dc8f452fb1aee77c3b336665 2024-12-08T21:17:12,492 INFO [M:0;e653e208f502:45981 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00af7895dc8f452fb1aee77c3b336665 2024-12-08T21:17:12,511 DEBUG [M:0;e653e208f502:45981 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/70610c6730fa460abe43d320d2ffa549 is 69, key is e653e208f502,38457,1733692444075/rs:state/1733692446999/Put/seqid=0 2024-12-08T21:17:12,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073742452_1628 (size=5156) 2024-12-08T21:17:12,915 INFO [M:0;e653e208f502:45981 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2084 (bloomFilter=true), to=hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/70610c6730fa460abe43d320d2ffa549 2024-12-08T21:17:12,926 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b732dbe249ff42179be1b143de7c9d7f as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b732dbe249ff42179be1b143de7c9d7f 2024-12-08T21:17:12,931 INFO [M:0;e653e208f502:45981 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b732dbe249ff42179be1b143de7c9d7f, entries=8, sequenceid=2084, filesize=5.5 K 2024-12-08T21:17:12,932 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/00af7895dc8f452fb1aee77c3b336665 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00af7895dc8f452fb1aee77c3b336665 2024-12-08T21:17:12,936 INFO [M:0;e653e208f502:45981 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 00af7895dc8f452fb1aee77c3b336665 2024-12-08T21:17:12,936 INFO [M:0;e653e208f502:45981 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/00af7895dc8f452fb1aee77c3b336665, entries=166, sequenceid=2084, filesize=43.7 K 2024-12-08T21:17:12,937 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/70610c6730fa460abe43d320d2ffa549 as hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/70610c6730fa460abe43d320d2ffa549 2024-12-08T21:17:12,940 INFO [M:0;e653e208f502:45981 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36411/user/jenkins/test-data/b954fc98-a215-8411-aa23-d5810ba8e6cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/70610c6730fa460abe43d320d2ffa549, entries=1, sequenceid=2084, filesize=5.0 K 2024-12-08T21:17:12,941 INFO [M:0;e653e208f502:45981 {}] regionserver.HRegion(3140): Finished flush of dataSize ~756.89 KB/775053, heapSize ~925.64 KB/947856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=2084, compaction requested=false 2024-12-08T21:17:12,943 INFO [M:0;e653e208f502:45981 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-08T21:17:12,943 DEBUG [M:0;e653e208f502:45981 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733692631625Disabling compacts and flushes for region at 1733692631625Disabling writes for close at 1733692631625Obtaining lock to block concurrent updates at 1733692631625Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733692631625Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=775053, getHeapSize=948096, getOffHeapSize=0, getCellsCount=2275 at 1733692631626 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733692631627 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733692631628 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733692631645 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733692631645Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733692632064 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733692632081 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733692632081Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733692632492 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733692632510 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733692632510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ac2646: reopening flushed file at 1733692632925 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50f135a8: reopening flushed file at 1733692632932 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76a4f98c: reopening flushed file at 1733692632936 (+4 ms)Finished flush of dataSize ~756.89 KB/775053, heapSize ~925.64 KB/947856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1316ms, sequenceid=2084, compaction requested=false at 1733692632941 (+5 ms)Writing region close event to WAL at 1733692632942 (+1 ms)Closed at 1733692632942 2024-12-08T21:17:12,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42767 is added to blk_1073741830_1006 (size=911696) 2024-12-08T21:17:12,946 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-08T21:17:12,946 INFO [M:0;e653e208f502:45981 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-08T21:17:12,946 INFO [M:0;e653e208f502:45981 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45981 2024-12-08T21:17:12,946 INFO [M:0;e653e208f502:45981 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-08T21:17:13,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:17:13,097 INFO [M:0;e653e208f502:45981 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-08T21:17:13,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45981-0x10007474e6b0000, quorum=127.0.0.1:56209, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-08T21:17:13,107 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-08T21:17:13,110 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T21:17:13,110 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T21:17:13,110 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T21:17:13,110 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/hadoop.log.dir/,STOPPED} 2024-12-08T21:17:13,113 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-08T21:17:13,113 WARN [BP-1093270908-172.17.0.2-1733692438514 heartbeating to localhost/127.0.0.1:36411 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-08T21:17:13,113 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-08T21:17:13,113 WARN [BP-1093270908-172.17.0.2-1733692438514 heartbeating to localhost/127.0.0.1:36411 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1093270908-172.17.0.2-1733692438514 (Datanode Uuid 201a3fb0-05db-4bd4-b37f-702c1891983f) service to localhost/127.0.0.1:36411 2024-12-08T21:17:13,115 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611/data/data1/current/BP-1093270908-172.17.0.2-1733692438514 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T21:17:13,116 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/cluster_57f49de5-43fa-4aaa-de44-341b485cc611/data/data2/current/BP-1093270908-172.17.0.2-1733692438514 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-08T21:17:13,116 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-08T21:17:13,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-08T21:17:13,125 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-08T21:17:13,125 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-08T21:17:13,125 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-08T21:17:13,125 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/54271f81-ea08-fd65-b112-4e38f2ded1bd/hadoop.log.dir/,STOPPED} 2024-12-08T21:17:13,140 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-08T21:17:13,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down